Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

feat!: Write and Insert no longer act on computer resources; Read only looks (LPT-934)

alexshpunt-clanker[bot]7ed711b7ed8d
diff --git a/src/computer-address.ts b/src/computer-address.ts
index 2f92c27..76bec2b 100644
--- a/src/computer-address.ts
+++ b/src/computer-address.ts
@@ -5,12 +5,13 @@ export type ComputerAddress =
| { kind: "session"; source: string }
/** A machine by its name: `local` for this one, otherwise a name under `machines` in .pi/computer-use.json. */
| { kind: "machine"; source: string; ref: string; part?: MachinePart }
- | { kind: "app" | "window" | "sequence" | "window-state" | "sandbox"; source: string; ref: string }
+ | { kind: "app" | "window" | "sequence" | "sandbox"; source: string; ref: string }
/** A display of one machine: `primary`, or a reference listed by that machine's displays. */
| { kind: "display"; source: string; machine: string; ref: string }
- | { kind: "observation"; source: string; ref: string; hint?: string; delivery?: "background" | "foreground"; delayMs?: number };
+ /** `hint` names one element of the observation for an internal input step; parsed addresses never carry it. */
+ | { kind: "observation"; source: string; ref: string; hint?: string };
-const references = new Set(["app", "window", "observation", "sequence", "window-state", "sandbox", "machine"]);
+const references = new Set(["app", "window", "observation", "sequence", "sandbox", "machine"]);
/** Words a long address starts with; machine names and window names never take them. */
export const ADDRESS_KINDS: ReadonlySet<string> = new Set([...references, "session", "display"]);
const parts = new Set(["windows", "apps", "displays"]);
@@ -42,27 +43,13 @@ export function parseComputerAddress(source: string): ComputerAddress | undefine
const machinePart = kind === "machine" && segments.length === 3 ? segments[2] : undefined;
if (!references.has(kind) || (segments.length !== 2 && !(machinePart && parts.has(machinePart)))) return invalid();
const ref = reference(segments[1]);
- if (kind !== "observation") {
- if (query !== undefined) return invalid();
- if (kind === "machine") return { kind, ref, source, ...(machinePart ? { part: machinePart as MachinePart } : {}) };
- return { kind: kind as "app" | "window" | "sequence" | "window-state" | "sandbox", ref, source };
- }
- const result: Extract<ComputerAddress, { kind: "observation" }> = { kind, ref, source };
- const params = new URLSearchParams(query);
- const seen = new Set<string>();
- for (const [key, value] of params) {
- if (seen.has(key)) return invalid();
- seen.add(key);
- if (key === "hint" && /^[a-z0-9]+:[a-z]+$/.test(value)) result.hint = value;
- else if (key === "delivery" && (value === "background" || value === "foreground")) result.delivery = value;
- else if (key === "delay_ms" && /^(0|[1-9][0-9]*)$/.test(value) && Number(value) <= 200) result.delayMs = Number(value);
- else return invalid();
- }
- return result;
+ if (query !== undefined) return invalid();
+ if (kind === "machine") return { kind, ref, source, ...(machinePart ? { part: machinePart as MachinePart } : {}) };
+ return { kind: kind as "app" | "window" | "observation" | "sequence" | "sandbox", ref, source };
}
/** Build a computer address from an opaque session-issued reference, or a machine's address from its name. */
-export function computerSource(kind: "app" | "window" | "observation" | "sequence" | "window-state" | "sandbox" | "machine", ref: string, part?: MachinePart): string {
+export function computerSource(kind: "app" | "window" | "observation" | "sequence" | "sandbox" | "machine", ref: string, part?: MachinePart): string {
const source = `computer:${kind}/${encodeURIComponent(ref)}${part ? `/${part}` : ""}`;
parseComputerAddress(source);
return source;
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 0f92562..22a477c 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -11,7 +11,6 @@ import { basename, dirname, resolve } from "node:path";
import type { ComputerSession, ComputerResult, ComputerMode, Action } from "./session.ts";
import { parseWindowManagement, type WindowManagementCommand } from "./window-management.ts";
import { validateWebAction, validateWebLaunch, type WebAction } from "./web-browser.ts";
-import { combine, evaluateNative, type WaitSpec } from "./ui-conditions.ts";
import { compareCaptures, diffDom, diffIsEmpty, diffUi, type Capture, type StateDiff } from "./successor.ts";
import { changeCrops, decodeFrame } from "./pixel-diff.ts";
import type { WindowFrame } from "./session.ts";
@@ -19,14 +18,13 @@ import type { UiCaptureOptions, UiElement, UiResource } from "./ui-resources.ts"
import { displayMode } from "./display-mode.ts";
import { isSemanticAction } from "./semantic-actions.ts";
import { accessibleName, ariaRole, checkedState, isVisible, nativeAriaSnapshot, nativeSelector, queryNative, suggestLocator, type ParsedSelector } from "./native-locator.ts";
-import { SETUP_PARTS, SetupError, runSetup, type SetupPart } from "./setup.ts";
/** The machine this extension runs on, by its address name. */
export const LOCAL_MACHINE = "local";
/** How to use a machine address: the same on every machine. */
export function machineGuide(source: string): string[] {
- return [`Search ${source}/windows for its windows and ${source}/apps for its installed apps; Read ${source}/displays for its displays and ${source}/display/primary for its main display (Insert resize into a display address to change its resolution).`,
- `Insert open, open_browser, open_sandbox, open_session, webview2_debugging, put, get, files, clipboard_read, clipboard_write, record_start, record_stop, audio_record, name (a window by its title) or close into ${source}.`];
+ return [`Search ${source}/windows for its windows and ${source}/apps for its installed apps; Read ${source}/displays for its displays and ${source}/display/primary for its main display (computer_display changes its resolution).`,
+ `Give ${source} to computer_open, computer_files, computer_clipboard and computer_record; Delete it to release it.`];
}
/** A rejected request whose validation completed before native dispatch. */
export class ComputerRequestError extends Error {}
@@ -44,10 +42,6 @@ function commandObject(value: string): Record<string, unknown> {
catch { throw new ComputerRequestError("Use one valid JSON command object."); }
}
-/** Steps of one mixed batch; each native step after the first walks the UI again, so batches stay shorter than sequences. */
-const BATCH_STEPS = 200;
-/** Native actions a batch may run; each later step re-resolves its target in a fresh UI walk. */
-const NATIVE_BATCH_ACTIONS = ["click", "double_click", "right_click", "type_text", "key", "hotkey", "scroll", "invoke", "select", "set_value"];
/** Native actions that take a Playwright locator or JSON target. */
const NATIVE_LOCATOR_ACTIONS = ["click", "double_click", "right_click", "hover", "type_text", "key", "hotkey", "scroll", "invoke", "select", "set_value", "check", "uncheck"];
/** A point in pixels of a window's latest image, or in CSS pixels of a browser page. */
@@ -84,11 +78,10 @@ export function refKind(target: unknown): "hint" | "page" | "locator" | undefine
/** Playwright verbs with a different native name. */
const LOCATOR_VERBS: Record<string, string> = { fill: "set_value", dblclick: "double_click" };
/**
- * What follows an action: its optional postcondition, and the successor with its change list. The successor is on
- * by default; `explicit` means the agent asked for it (`successor: true` or `expect`), which also overrides the
- * defaults that skip it. `frame` holds the window's pixels before the action, when the input observation had them.
+ * What follows an action: the successor with its change list. `explicit` overrides the defaults that skip it.
+ * `frame` holds the window's pixels before the action, when the input observation had them.
*/
-interface FollowUp { expect?: WaitSpec; successor: boolean; explicit: boolean; preexisting?: boolean; frame?: WindowFrame;
+interface FollowUp { successor: boolean; explicit: boolean; frame?: WindowFrame;
/** The window's session, resolved before the action: a failed action can make the session forget the window. */
session?: ComputerSession }
/** How long a native successor keeps looking for a change before reporting that nothing changed. */
@@ -266,30 +259,6 @@ async function fileCommand(session: ComputerSession, command: Record<string, unk
throw new ComputerRequestError(`File ${command.action} failed: ${error instanceof Error ? error.message : error}`);
}
}
-/** Install an optional part on a machine, or prepare another one from this machine; the user must have allowed it. */
-async function setupCommand(command: Record<string, unknown>, machine: string | undefined, cwd: string, mode: ComputerMode, signal?: AbortSignal): Promise<ComputerResult> {
- only(command, ["action", "part", "ssh", "name", "code", "argv"]);
- const part = command.part;
- if (typeof part !== "string" || !SETUP_PARTS.some(item => item.name === part))
- throw new ComputerRequestError(`setup needs part: one of ${SETUP_PARTS.map(item => item.name).join(", ")}. Install only after the user explicitly allowed that part on that machine.`);
- const strings: Record<string, string> = {};
- for (const key of ["ssh", "name", "code", "argv"]) {
- if (command[key] === undefined) continue;
- if (typeof command[key] !== "string" || !command[key]) throw new ComputerRequestError(`setup ${key} must be a non-empty string.`);
- strings[key] = command[key] as string;
- }
- if (part === "machine" && machine) throw new ComputerRequestError("Prepare another machine from computer:machine/local.");
- if (part !== "machine" && (strings.ssh || strings.name)) throw new ComputerRequestError("ssh and name belong to setup part machine.");
- if (part !== "vscode-bridge" && (strings.code || strings.argv)) throw new ComputerRequestError("code and argv belong to setup part vscode-bridge.");
- try {
- const text = await runSetup({ part: part as SetupPart, ...(machine ? { machine } : {}), ...strings }, cwd, { ...(signal ? { signal } : {}) });
- return { content: [{ type: "text", text }], details: { status: "delivered", mode, state: { operation: "setup", part, machine: machine ?? LOCAL_MACHINE } } };
- } catch (error) {
- if (error instanceof SetupError) throw new ComputerRequestError(error.message);
- return { content: [{ type: "text", text: `Setup of ${part} failed: ${error instanceof Error ? error.message : error}` }],
- details: { status: "refused", mode, state: { operation: "setup", part, machine: machine ?? LOCAL_MACHINE } }, isError: true };
- }
-}
const unresponsive = (health: ProcessHealth) => `The app ${health.name} (pid ${health.pid}) is not responding; it uses ${health.memory_mb} MB memory. Capture and input cannot recover a hung app: wait and read again, or close it.`;
function object(value: unknown): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) throw new ComputerRequestError("Use a JSON command object.");
@@ -297,10 +266,6 @@ function object(value: unknown): Record<string, unknown> {
}
/** A window name the agent gives: computer:<name>. */
const NAME = /^[a-z][a-z0-9_-]{0,31}$/;
-/** Whether an Insert text is a name command, checked before the address's other commands. */
-function nameCommand(value: string): boolean {
- try { return (JSON.parse(value) as { action?: unknown })?.action === "name"; } catch { return false; }
-}
function only(value: Record<string, unknown>, fields: readonly string[]): void {
if (Object.keys(value).some(key => !fields.includes(key))) throw new ComputerRequestError("Command contains unsupported fields.");
}
@@ -308,7 +273,6 @@ function only(value: Record<string, unknown>, fields: readonly string[]): void {
/** Addressable projections of one ComputerSession. Cached descriptors never acquire native pixels. */
export class ComputerResources {
private readonly snapshots = new Map<string, Snapshot>();
- private readonly windowStates = new Map<string, { window_ref: string; result: ComputerResult }>();
/** Each window's latest observed UI, the baseline of its next `changes` Read. Outlives the observation's input authority. */
private readonly lastUi = new Map<string, { observation_id: string; ui: UiResource }>();
/** Installed apps by opaque reference, each with the machine that listed it. */
@@ -380,9 +344,9 @@ export class ComputerResources {
const base = at < 0 ? source : source.slice(0, at), query = at < 0 ? "" : source.slice(at);
const segments = base.slice(9).split("/");
if (segments.length === 1 && !ADDRESS_KINDS.has(segments[0]) && NAME.test(segments[0])) {
- if (query) throw new ComputerRequestError("A window name takes no options. Read it for an observation and put options on that observation's address.");
+ if (query) throw new ComputerRequestError("A window name takes no options.");
const entry = this.names.get(segments[0]);
- if (!entry) throw new ComputerRequestError(`No window is named ${segments[0]}. Insert {"action":"name","as":"${segments[0]}"} into a window address first; Read computer:session for the current names.`);
+ if (!entry) throw new ComputerRequestError(`No window is named ${segments[0]}. Name a window with computer_window name first; Read computer:session for the current names.`);
if (!this.holdsWindow(entry.window_ref)) {
this.names.delete(segments[0]);
throw new ComputerRequestError(`computer:${segments[0]} named a window that has closed or was released${entry.title ? ` (${entry.title})` : ""}. Search the machine's windows again and name the new one.`);
@@ -408,15 +372,10 @@ export class ComputerResources {
// That machine's session refuses a display reference it did not list, after its own mode check, without native work.
return { session: await this.machineSession(cwd, target.machine, signal), machine: target.machine };
}
- /** The owner of a window, display, window state, observation or sequence address. */
+ /** The owner of a window, display, observation or sequence address. */
private async routeAddress(cwd: string, address: ComputerAddress, signal?: AbortSignal): Promise<Owner> {
if (address.kind === "window") return this.route(cwd, { window_ref: address.ref }, signal);
if (address.kind === "display") return this.route(cwd, { display: address.ref, machine: address.machine }, signal);
- if (address.kind === "window-state") {
- const state = this.windowStates.get(address.ref);
- if (!state) throw new ComputerRequestError("Window state reference expired. Read the window with state view again.");
- return this.route(cwd, { window_ref: state.window_ref }, signal);
- }
if (address.kind === "observation") return this.route(cwd, this.snapshot(address.ref).target, signal);
if (address.kind === "sequence")
return this.owner(cwd, session => session.hasSequence(address.ref), "Sequence is unavailable or evicted. Capture again; sequences do not survive reload or close.");
@@ -437,7 +396,6 @@ export class ComputerResources {
case "sandbox": { const sandbox = this.sandboxes.get(address.ref); return sandbox && { machine: sandbox.machine ?? LOCAL_MACHINE, sandbox: sandbox.ref }; }
case "sequence": return holder(session => session.hasSequence(address.ref));
case "window": return holder(session => session.hasWindow(address.ref));
- case "window-state": { const state = this.windowStates.get(address.ref); return state && holder(session => session.hasWindow(state.window_ref)); }
case "observation": {
const target = this.snapshots.get(address.ref)?.target;
return !target ? undefined : target.machine !== undefined ? { machine: target.machine } : holder(session => session.hasWindow(target.window_ref!));
@@ -470,11 +428,10 @@ export class ComputerResources {
private async machineSession(cwd: string, name: string, signal?: AbortSignal): Promise<ComputerSession> {
return name === LOCAL_MACHINE ? (this.local = await this.getSession(cwd)) : (await this.machine(cwd, name, signal)).session;
}
- /** Forget a released session's observations and window states, and the display observations of its machine. */
+ /** Forget a released session's observations, and the display observations of its machine. */
private forget(session: ComputerSession, machine?: string): void {
for (const [id, entry] of this.snapshots)
if (entry.target.window_ref !== undefined ? session.hasWindow(entry.target.window_ref) : entry.target.machine === machine) this.snapshots.delete(id);
- for (const [id, entry] of this.windowStates) if (session.hasWindow(entry.window_ref)) this.windowStates.delete(id);
}
/**
* Release one machine: its sandboxes and sessions, then its own session with its browsers and file folder. The
@@ -488,7 +445,6 @@ export class ComputerResources {
if (name === LOCAL_MACHINE) {
for (const [id, entry] of this.snapshots)
if (entry.target.window_ref !== undefined ? !this.remoteWindow(entry.target.window_ref) : entry.target.machine === LOCAL_MACHINE) this.snapshots.delete(id);
- for (const [id, entry] of this.windowStates) if (!this.remoteWindow(entry.window_ref)) this.windowStates.delete(id);
own = await this.closeSession().catch(error => failure(error, this.getMode()));
} else {
const machine = this.machines.get(name);
@@ -542,7 +498,6 @@ export class ComputerResources {
const names = [...new Set([...this.machines.keys(), ...[...this.sandboxes.values()].map(sandbox => sandbox.machine ?? LOCAL_MACHINE)])].filter(name => name !== LOCAL_MACHINE);
const results = await Promise.all([...names.map(name => this.closeMachine(name)), this.closeMachine(LOCAL_MACHINE)]);
this.snapshots.clear();
- this.windowStates.clear();
this.apps.clear();
return report(this.getMode(), results.map(result => [undefined, result]), "Computer use released. Application documents were not closed or saved.");
}
@@ -554,7 +509,6 @@ export class ComputerResources {
}
/** Forget a window's observations: all of them, or only its DOM snapshots or only its native captures, which a Read of the other kind keeps. */
private retireWindow(ref: string, kind?: "dom" | "native"): void {
- for (const [id, entry] of this.windowStates) if (entry.window_ref === ref) this.windowStates.delete(id);
for (const [id, entry] of this.snapshots) if (entry.target.window_ref === ref && (!kind || observationKind(entry.result) === kind)) this.snapshots.delete(id);
}
private remember(target: ObservationTarget, result: ComputerResult): ComputerResult {
@@ -575,26 +529,19 @@ export class ComputerResources {
return result;
}
- /** Whether an observation or window-state address still grants its one-use authority. Other addresses grant none. */
+ /** Whether an observation address is still current for input. Other addresses grant none. */
grantsAuthority(address: ComputerAddress): boolean {
- return address.kind === "observation" ? this.snapshots.has(address.ref)
- : address.kind === "window-state" ? this.windowStates.has(address.ref) : false;
+ return address.kind === "observation" && this.snapshots.has(address.ref);
}
/** Resolve an editor's readonly descriptor without native acquisition or target refresh. */
descriptor(address: ComputerAddress): string {
- if (address.kind === "window-state") {
- const state = this.windowStates.get(address.ref);
- if (!state) throw new ComputerRequestError("Window state reference expired. Read the window with state view again.");
- const operations = state.result.details.window_state_kind === "management" ? "move/resize/minimize/maximize/focus or restore" : "restore";
- return `# Computer window state\nWindow: ${computerSource("window", state.window_ref)}\nState: ${address.source}\nInsert one ${operations} command consumes this state. No capture, typing or pointer authority. No native query occurred during descriptor resolution.`;
- }
if (address.kind === "observation") {
const snapshot = this.snapshot(address.ref);
const source = snapshot.target.window_ref !== undefined ? computerSource("window", snapshot.target.window_ref)
: displaySource(snapshot.target.machine!, snapshot.target.display!);
return `# Computer observation\nSource: ${source}\nObservation: ${computerSource("observation", address.ref)}\n` +
- (snapshot.result.details.dom ? "Playwright DOM snapshot. Insert one JSON action with a target (aria ref or locator); Read element:crop|highlight,<ref> or image for page pixels. Search filters this snapshot. Write and native input are unavailable here."
- : snapshot.target.window_ref !== undefined ? "Input needs this still-current observation. Insert sends JSON commands; Write sends literal text."
+ (snapshot.result.details.dom ? "Playwright DOM snapshot. Its refs are targets for the computer_* input tools on its window; Read element:crop|highlight,<ref> or image for page pixels. Search filters this snapshot."
+ : snapshot.target.window_ref !== undefined ? "Its refs are targets for the computer_* input tools on its window until input spends them."
: "Display observation is readonly and grants no input authority.");
}
if (address.kind === "sandbox") return this.sandbox(address.ref).describe(address.source);
@@ -613,15 +560,24 @@ export class ComputerResources {
const sandboxes = [...this.sandboxes.values()].map(sandbox => `- ${computerSource("sandbox", sandbox.ref)} on ${computerSource("machine", sandbox.machine ?? LOCAL_MACHINE)}`);
return ["# Computer use", "Machines (computer:machine/local is this one; a machine configured under machines in .pi/computer-use.json starts on its first use). Every window, display, observation and sequence address belongs to the machine that issued it, and every receipt names that machine:",
...machines, ...(sandboxes.length ? ["Sandboxes and isolated sessions:", ...sandboxes] : []),
- ...(names.length ? ["Window names (Insert {\"action\":\"name\",\"as\":\"<name>\"} into a window address to add one):", ...names] : []),
+ ...(names.length ? ["Window names (computer_window name adds one):", ...names] : []),
...(windows.length ? ["Windows listed so far:", ...windows] : []),
- "Insert {\"action\":\"close\"} here to release every machine, browser, sandbox and session this computer use started; the result lists each part."].join("\n");
+ "Delete this address to release every machine, browser, sandbox and session this computer use started; the result lists each part."].join("\n");
}
- return `# Computer resource\nSource: ${address.source}\nUse Read for native observation, Search for discovery, and Insert for explicit commands. No native capture occurred during descriptor resolution.`;
+ return `# Computer resource\nSource: ${address.source}\nUse Read to look, Search to discover, and the computer_* tools to act. No native capture occurred during descriptor resolution.`;
}
/** Acquire a requested native projection, or inspect one explicitly issued observation; the receipt names its machine. */
async read(address: ComputerAddress, cwd: string, views?: readonly string[], signal?: AbortSignal): Promise<ComputerResult | undefined> {
+ if (parseComputerViews(views).kind === "sequence") throw new ComputerRequestError("Read only looks; capture frames over time with computer_record action sequence.");
+ return this.look(address, cwd, views, signal);
+ }
+ /** Capture frames of a window or display over time for computer_record; the sequence address then takes frame Reads. */
+ sequence(address: ComputerAddress, cwd: string, views: readonly string[], signal?: AbortSignal): Promise<ComputerResult | undefined> {
+ if (parseComputerViews(views).kind !== "sequence") throw new ComputerRequestError("computer_record sequence takes sequence options only.");
+ return this.look(address, cwd, views, signal);
+ }
+ private async look(address: ComputerAddress, cwd: string, views?: readonly string[], signal?: AbortSignal): Promise<ComputerResult | undefined> {
const place = this.place(address);
const result = await this.readAddress(address, cwd, views, signal);
return result && this.stamp(place, result);
@@ -711,23 +667,10 @@ export class ComputerResources {
}
const owner = await this.routeAddress(cwd, address, signal);
const session = owner.session;
- if (address.kind === "window-state") {
- if (views?.length) throw new ComputerRequestError("Cached window state has no capture or UI views. Read the window with state view for a fresh state.");
- const state = this.windowStates.get(address.ref);
- if (!state) throw new ComputerRequestError("Window state reference expired. Read the window with state view again.");
- const checked = await session.checkWindowState(state.window_ref, address.ref, signal);
- if (checked.details.status !== "observed") { this.windowStates.delete(address.ref); return checked; }
- return { ...state.result, content: state.result.content.map(block => block.type === "text"
- ? { ...block, text: block.text.replace(/^.*?(?=\n|$)/,
- "Cached native window state. No new metadata sample or renewed one-use authority. Insert consumes this state once.") }
- : block) };
- }
if (view.kind === "sequence") {
if (address.kind !== "window" && address.kind !== "display") throw new ComputerRequestError("Capture a sequence from a window/display address, not an observation.");
if (owner.sandbox) throw new ComputerRequestError("Temporal capture is not available for sandbox windows.");
const target: ObservationTarget = address.kind === "display" ? { display: address.ref, machine: address.machine } : { window_ref: address.ref };
- if (target.window_ref) for (const [id, entry] of this.windowStates)
- if (entry.window_ref === target.window_ref) this.windowStates.delete(id);
for (const [id, entry] of this.snapshots) if (sameTarget(entry.target, target)) this.snapshots.delete(id);
return session.captureSequence(address.kind === "window" ? { window_ref: address.ref } : { display: address.ref }, view.options, signal);
}
@@ -738,15 +681,9 @@ export class ComputerResources {
if (view.kind === "frames") throw new ComputerRequestError("Frame selection needs a sequence address.");
if (address.kind === "window") {
// A fresh window Read retires the earlier observation of its kind; a DOM snapshot and a native capture coexist.
- this.retireWindow(address.ref, view.kind === "dom" ? "dom" : view.kind === "state" ? undefined : "native");
- if (view.kind === "state") {
- const result = await session.observeWindowState(address.ref, signal);
- if (result.details.status === "observed" && result.details.window_state_id) {
- this.windowStates.set(result.details.window_state_id, { window_ref: address.ref, result });
- while (this.windowStates.size > 64) this.windowStates.delete(this.windowStates.keys().next().value!);
- }
- return result;
- }
+ // A state Read only looks and keeps the window's observations; computer_window samples its own state for each change.
+ if (view.kind === "state") return session.observeWindowState(address.ref, signal);
+ this.retireWindow(address.ref, view.kind === "dom" ? "dom" : "native");
if (view.kind === "dom") return this.remember({ window_ref: address.ref }, await session.observeDom(address.ref, view.options, signal));
if (view.kind !== "overview") throw new ComputerRequestError("Read an overview, then apply region to its returned observation address.");
return this.remember({ window_ref: address.ref }, await session.observe(address.ref, signal, undefined, undefined, undefined, view.capture));
@@ -760,8 +697,6 @@ export class ComputerResources {
}
if (address.kind !== "observation") throw new ComputerRequestError("Use an existing supported computer resource.");
const snapshot = this.snapshot(address.ref);
- if (address.hint || address.delivery || address.delayMs !== undefined)
- throw new ComputerRequestError("Hint/delivery/delay address options are for input, not Read.");
if (snapshot.result.details.dom) {
const ref = snapshot.target.window_ref;
if (!ref) throw new ComputerRequestError("Readonly DOM requires a window source.");
@@ -847,7 +782,6 @@ export class ComputerResources {
}
if (address.kind !== "observation") throw new ComputerRequestError("Search computer:machine/<name>/windows, computer:machine/<name>/apps, a sandbox address or a returned observation address.");
const session = (await this.routeAddress(cwd, address, signal)).session;
- if (address.hint || address.delivery || address.delayMs !== undefined) throw new ComputerRequestError("Input options do not select a Search scope.");
const snapshot = this.snapshot(address.ref);
if (!snapshot.target.window_ref) throw new ComputerRequestError("Display images have no native UI search scope.");
if (snapshot.result.details.dom) return session.searchDom(snapshot.target.window_ref, address.ref, query, limit, signal);
@@ -957,7 +891,7 @@ export class ComputerResources {
try { action = validateWebAction(command); }
catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }
const dom = session.inputView(window).dom;
- if (dom && this.snapshots.has(dom)) return this.performAt({ kind: "observation", ref: dom, source: computerSource("observation", dom) }, cwd, JSON.stringify(command), "insert", signal);
+ if (dom && this.snapshots.has(dom)) return this.performAt({ kind: "observation", ref: dom, source: computerSource("observation", dom) }, cwd, JSON.stringify(command), signal);
this.retireWindow(window);
const result = await session.actPage(window, action, signal);
return result.details.dom ? this.remember({ window_ref: window }, result) : result;
@@ -990,7 +924,7 @@ export class ComputerResources {
let retry = (): InputPlan => plan;
if (refKind(target) === "locator") {
if (plan.end) throw new ComputerRequestError(`${plan.label} takes refs of the window's latest Read or points, not locators.`);
- result = await this.performAt({ kind: "window", ref: window, source: computerSource("window", window) }, cwd, JSON.stringify({ ...command, target }), "insert", signal);
+ result = await this.performAt({ kind: "window", ref: window, source: computerSource("window", window) }, cwd, JSON.stringify({ ...command, target }), signal);
} else {
const hint = refKind(target) === "hint" ? target as string : undefined;
// Only a locator waits for its element; refs and points act on the current observation at once.
@@ -1027,7 +961,7 @@ export class ComputerResources {
if (hint && action !== "drag") retry = () => NATIVE_LOCATOR_ACTIONS.includes(LOCATOR_VERBS[action] ?? action) ? { ...plan, target: suggestLocator(this.hinted(view, hint)) }
: { ...plan, target: undefined, point: seen(centre(hint)) };
const observation = { kind: "observation" as const, ref: native.id, source: computerSource("observation", native.id), ...(hint && action !== "drag" ? { hint } : {}) };
- result = await this.performAt(observation, cwd, JSON.stringify(command), "insert", signal);
+ result = await this.performAt(observation, cwd, JSON.stringify(command), signal);
}
const capability = result.details.action_capability;
if (foreground || plan.foreground === false || result.details.status !== "refused" || capability?.background !== "unavailable") return result;
@@ -1042,62 +976,23 @@ export class ComputerResources {
if (!element) throw new ComputerRequestError(`${hint} is not a ref of this window's latest Read, or input since then spent it: Read the window again, or give a Playwright locator.`);
return element;
}
- /** Dispatch exactly one literal Write or JSON Insert against its address-bound target; the receipt names its machine. */
- async perform(address: ComputerAddress, cwd: string, value: string, tool: "write" | "insert", signal?: AbortSignal): Promise<ComputerResult> {
+ /** Dispatch exactly one JSON command of a computer_* tool against its address-bound target; the receipt names its machine. */
+ async perform(address: ComputerAddress, cwd: string, value: string, signal?: AbortSignal): Promise<ComputerResult> {
const place = this.place(address);
- return this.stamp(place, await this.performAt(address, cwd, value, tool, signal));
+ return this.stamp(place, await this.performAt(address, cwd, value, signal));
}
- private async performAt(address: ComputerAddress, cwd: string, value: string, tool: "write" | "insert", signal?: AbortSignal): Promise<ComputerResult> {
- if (tool === "insert" && (address.kind === "window" || address.kind === "machine" && !address.part) && nameCommand(value)) {
- const command = commandObject(value);
- if (address.kind === "window") {
- only(command, ["action", "as"]);
- await this.route(cwd, { window_ref: address.ref }, signal);
- return this.nameWindow(command.as, address.ref);
- }
- only(command, ["action", "as", "title"]);
- if (typeof command.title !== "string" || !command.title) throw new ComputerRequestError("Name a window of a machine by its title: {\"action\":\"name\",\"as\":\"game\",\"title\":\"<text or regex:...>\"}, or Insert the name command into the window's address.");
- const listed = await this.search(requireComputerAddress(computerSource("machine", address.ref, "windows")), cwd, command.title, undefined, signal);
- const rows = ((listed.details.state?.windows ?? []) as { window_ref?: string; title?: string }[]).filter(row => typeof row.window_ref === "string");
- if (rows.length !== 1) {
- const candidates = rows.slice(0, 20).map(row => `- ${computerSource("window", row.window_ref!)} | ${row.title ?? ""}`);
- throw new ComputerRequestError(rows.length ? `${rows.length} windows match ${JSON.stringify(command.title)}; name one by its address or a narrower title:\n${candidates.join("\n")}`
- : `No window of ${computerSource("machine", address.ref)} matches ${JSON.stringify(command.title)}.`);
- }
- return this.nameWindow(command.as, rows[0].window_ref!);
- }
- if (address.kind === "window-state") {
- if (tool !== "insert") throw new ComputerRequestError("Window state grants native control through Insert, never literal Write.");
- const command = commandObject(value);
- let management: WindowManagementCommand | undefined;
- if (command.action === "restore") only(command, ["action"]);
- else {
- try { management = parseWindowManagement(command); }
- catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }
- }
- const state = this.windowStates.get(address.ref);
- if (!state) throw new ComputerRequestError("Window state reference expired. Read the window with state view again.");
- this.windowStates.delete(address.ref);
- const session = await this.sessionFor(cwd, { window_ref: state.window_ref });
- return management ? session.manageWindow(state.window_ref, address.ref, management, signal)
- : session.restoreWindow(state.window_ref, address.ref, signal);
- }
+ private async performAt(address: ComputerAddress, cwd: string, value: string, signal?: AbortSignal): Promise<ComputerResult> {
if (address.kind === "sandbox") {
const sandbox = this.sandbox(address.ref);
- if (tool !== "insert") throw new ComputerRequestError("A sandbox address accepts Insert open or close, not literal Write.");
const command = commandObject(value);
- if (command.action === "close") { only(command, ["action"]); return this.closeSandbox(sandbox); }
- if (command.action !== "open") throw new ComputerRequestError("A sandbox address accepts {\"action\":\"open\",\"app\":\"...\"} or {\"action\":\"close\"}.");
+ if (command.action !== "open") throw new ComputerRequestError("A sandbox address opens apps.");
const result = await sandbox.session.open(openTarget(command, "in the sandbox"), signal);
return result.details.window_ref ? this.remember({ window_ref: result.details.window_ref }, result) : result;
}
if (address.kind === "machine") {
- if (tool !== "insert" || address.part) throw new ComputerRequestError(`Insert commands into ${computerSource("machine", address.ref)}, not literal Write or its windows, apps or displays.`);
+ if (address.part) throw new ComputerRequestError(`Give ${computerSource("machine", address.ref)} itself, not its windows, apps or displays.`);
const command = commandObject(value);
- if (command.action === "close") { only(command, ["action"]); return this.closeMachine(address.ref); }
const remote = address.ref === LOCAL_MACHINE ? undefined : address.ref;
- // Setup prepares a machine, so it must not start that machine's driver first.
- if (command.action === "setup") return setupCommand(command, remote, cwd, this.getMode(), signal);
if (command.action === "open_sandbox" || command.action === "open_session") {
// A remote machine starts before its sandbox, so an unknown name refuses first.
if (remote) await this.machine(cwd, remote, signal);
@@ -1119,24 +1014,18 @@ export class ComputerResources {
return result.details.window_ref ? this.remember({ window_ref: result.details.window_ref }, result) : result;
}
if (command.action !== "open")
- throw new ComputerRequestError(`A machine address accepts open, open_browser, open_sandbox, open_session, webview2_debugging, setup, ${FILE_ACTIONS.join(", ")} or close.`);
+ throw new ComputerRequestError(`A machine address accepts open, open_browser, open_sandbox, open_session, webview2_debugging or ${FILE_ACTIONS.join(", ")}.`);
const result = await session.open(openTarget(command, "on that machine"), signal);
return result.details.window_ref ? this.remember({ window_ref: result.details.window_ref }, result) : result;
}
- if (tool === "insert" && address.kind === "session") {
- const command = commandObject(value);
- if (command.action !== "close") throw new ComputerRequestError("computer:session accepts only close. Insert machine commands into computer:machine/local or computer:machine/<name>.");
- only(command, ["action"]);
- return this.closeAll();
- }
- if (tool === "insert" && address.kind === "app") {
+ if (address.kind === "app") {
only(commandObject(value), ["action"]);
const app = this.apps.get(address.ref);
if (!app) throw new ComputerRequestError("App reference expired. Search computer:machine/<name>/apps again.");
const result = await (await this.machineSession(cwd, app.machine, signal)).open(app.id, signal);
return result.details.window_ref ? this.remember({ window_ref: result.details.window_ref }, result) : result;
}
- if (address.kind === "display" && tool === "insert") {
+ if (address.kind === "display") {
const command = commandObject(value);
if (command.action !== "resize") throw new ComputerRequestError("A display address accepts only resize. Read it for pixels.");
only(command, ["action", "width", "height"]);
@@ -1151,26 +1040,19 @@ export class ComputerResources {
if (result.details.status !== "refused") { for (const id of stale) this.snapshots.delete(id); this.forget(session, address.machine); }
return result;
}
- if (address.kind === "window" && tool === "insert") {
+ if (address.kind === "window") {
const command = commandObject(value);
const session = await this.sessionFor(cwd, { window_ref: address.ref });
- if (command.action === "kill") {
- only(command, ["action"]);
- this.retireWindow(address.ref);
- return session.killOwned(address.ref, signal);
- }
if (command.action === "attach_browser") {
only(command, ["action"]);
return session.attachBrowser(address.ref, signal);
}
if (command.target !== undefined && command.action !== "wait_for" && command.action !== "verify") {
- if (session.isBrowserWindow(address.ref)) throw new ComputerRequestError("For a browser window, Read it with dom and Insert the action into that DOM observation; Playwright resolves the target there.");
- const follow = await this.followUpSpec(address.ref, command, cwd);
- delete command.expect; delete command.successor;
- return this.locatorAction(address.ref, session, command, cwd, signal, follow);
+ if (session.isBrowserWindow(address.ref)) throw new ComputerRequestError("A browser window's locators resolve in its page through Playwright.");
+ return this.locatorAction(address.ref, session, command, cwd, signal, { successor: true, explicit: false });
}
if (command.action !== "wait_for" && command.action !== "verify")
- throw new ComputerRequestError("A window address accepts an action with a target (a Playwright locator such as getByRole(\"button\", { name: \"Save\" })), wait_for, verify, attach_browser or kill. Other input needs the observation address returned by a fresh window Read.");
+ throw new ComputerRequestError("A window address accepts an action with a target (a Playwright locator such as getByRole(\"button\", { name: \"Save\" })), wait_for, verify or attach_browser.");
let spec;
try { spec = session.waitSpec(address.ref, command); }
catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }
@@ -1181,42 +1063,22 @@ export class ComputerResources {
if (address.kind !== "observation") throw new ComputerRequestError("Input needs the observation address returned by a fresh window Read.");
const snapshot = this.snapshot(address.ref);
const window = snapshot.target.window_ref;
- let input = value;
- let follow: FollowUp | undefined;
- if (tool === "insert" && window) {
- const command = commandObject(value);
- follow = await this.withFrame(await this.followUpSpec(window, command, cwd), window, address.ref, cwd);
- // Was the postcondition already true before anything is sent? Browsers sample now; native uses the input walk.
- if (follow.expect) {
- const session = await this.sessionFor(cwd, { window_ref: window });
- const earlier = snapshot.result.details.dom ? await session.webCheck(window, follow.expect.conditions, signal)
- : snapshot.result.details.ui ? evaluateNative(snapshot.result.details.ui, follow.expect.conditions) : undefined;
- follow.preexisting = earlier !== undefined && combine(earlier) === "satisfied";
- }
- if ("expect" in command || "successor" in command) {
- delete command.expect; delete command.successor;
- input = JSON.stringify(command);
- }
- if (command.action === "batch") {
- const result = await this.batch(address, snapshot, command, cwd, signal);
- return this.followUp(window, snapshot, result, follow, cwd, signal);
- }
- if (command.target !== undefined && !snapshot.result.details.dom) {
- if (address.hint !== undefined) throw new ComputerRequestError("Choose the address hint or a JSON target, not both.");
- this.snapshots.delete(address.ref);
- return this.locatorAction(window, await this.sessionFor(cwd, { window_ref: window }), command, cwd, signal, follow);
- }
- } else if (tool === "write" && window) follow = await this.withFrame({ successor: true, explicit: false }, window, address.ref, cwd);
- const result = await this.act(address, snapshot, input, tool, cwd, signal);
+ if (window && commandObject(value).target !== undefined && !snapshot.result.details.dom) {
+ if (address.hint !== undefined) throw new ComputerRequestError("Choose the ref or a locator target, not both.");
+ this.snapshots.delete(address.ref);
+ return this.locatorAction(window, await this.sessionFor(cwd, { window_ref: window }), commandObject(value), cwd, signal, { successor: true, explicit: false });
+ }
+ const follow = window ? await this.withFrame({ successor: true, explicit: false }, window, address.ref, cwd) : undefined;
+ const result = await this.act(address, snapshot, value, cwd, signal);
if (!follow || !window) return result;
return this.followUp(window, snapshot, result, follow, cwd, signal);
}
/** Dispatch one action against its observation: Playwright for DOM observations, CUA otherwise. */
- private async act(address: Extract<ComputerAddress, { kind: "observation" }>, snapshot: Snapshot, value: string, tool: "write" | "insert", cwd: string, signal?: AbortSignal): Promise<ComputerResult> {
+ private async act(address: Extract<ComputerAddress, { kind: "observation" }>, snapshot: Snapshot, value: string, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {
if (snapshot.result.details.dom) {
- if (tool !== "insert" || address.hint !== undefined || address.delivery !== undefined || address.delayMs !== undefined || !snapshot.target.window_ref)
- throw new ComputerRequestError("A DOM observation accepts one Insert JSON action with a target, not Write or address options.");
+ if (address.hint !== undefined || !snapshot.target.window_ref)
+ throw new ComputerRequestError("A page action takes its target in the command, not a native ref.");
let command;
try { command = validateWebAction(commandObject(value)); }
catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }
@@ -1225,24 +1087,13 @@ export class ComputerResources {
return result.details.dom ? this.remember({ window_ref: snapshot.target.window_ref }, result) : result;
}
if (!snapshot.target.window_ref) throw new ComputerRequestError("Display observations are readonly; choose a window overview for input.");
- let command: Record<string, unknown>;
- if (tool === "write") {
- command = { action: "type_text", text: value, ...(address.delayMs !== undefined ? { delay_ms: address.delayMs } : {}) };
- } else {
- if (address.delayMs !== undefined) throw new ComputerRequestError("delay_ms in the resource address is only for literal Write.");
- command = commandObject(value);
- if ("window_ref" in command || "observation_id" in command) throw new ComputerRequestError("The resource address binds the window and observation; omit identity fields from JSON.");
- }
+ const command = commandObject(value);
+ if ("window_ref" in command || "observation_id" in command) throw new ComputerRequestError("The window and observation come from the address; omit identity fields.");
if (address.hint !== undefined) {
if ("hint" in command || "element_token" in command || "x" in command || "y" in command)
throw new ComputerRequestError("Choose the address hint or a JSON target, not both.");
command.hint = address.hint;
}
- // The address delivery is a preference for pointer and keyboard input. Semantic actions always use accessibility.
- if (address.delivery !== undefined && !isSemanticAction(String(command.action))) {
- if ("delivery_mode" in command) throw new ComputerRequestError("Choose delivery in the address or JSON, not both.");
- command.delivery_mode = address.delivery;
- }
const action = { ...command, window_ref: snapshot.target.window_ref, observation_id: address.ref } as Action;
const session = await this.sessionFor(cwd, snapshot.target);
let result = await session.act(action, signal);
@@ -1265,23 +1116,7 @@ export class ComputerResources {
return { ...follow, session, ...(frame && frame.observation_id === observation ? { frame } : {}) };
}
- /** Validate an action's optional `expect` postcondition and `successor` request before anything is dispatched. */
- private async followUpSpec(window: string, command: Record<string, unknown>, cwd: string): Promise<FollowUp> {
- if (command.successor !== undefined && typeof command.successor !== "boolean") throw new ComputerRequestError("successor must be boolean.");
- let expect: WaitSpec | undefined;
- if (command.expect !== undefined) {
- const raw = command.expect;
- if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new ComputerRequestError("expect must be an object with conditions and optional timeout_ms, interval_ms and stable.");
- try { expect = (await this.sessionFor(cwd, { window_ref: window })).waitSpec(window, { ...raw as Record<string, unknown>, action: "wait_for" }); }
- catch (error) { throw new ComputerRequestError(`expect: ${error instanceof Error ? error.message : String(error)}`); }
- }
- return { ...(expect ? { expect } : {}), successor: command.successor !== false || expect !== undefined, explicit: command.successor === true || expect !== undefined };
- }
-
- /**
- * After an action, wait for its declared postcondition and attach the successor observation with a compact
- * diff from the action's input observation. A refused or unknown action never yields a verified postcondition.
- */
+ /** After an action, attach the successor observation with a compact diff from the action's input observation. */
private async followUp(window: string, before: Snapshot, result: ComputerResult, follow: FollowUp, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {
const status = result.details.status;
if (!follow.successor) return result;
@@ -1291,22 +1126,8 @@ export class ComputerResources {
// A cancelled call returns its receipt without capturing again.
if (signal?.aborted) return result;
const session = follow.session ?? await this.sessionFor(cwd, { window_ref: window });
- const preexisting = follow.preexisting === true;
let successor: ComputerResult | undefined, settled: StateDiff | undefined, crops: ComputerResult["content"] = [];
- let postcondition: ComputerResult["details"]["postcondition"];
- if (follow.expect && status !== "refused") {
- // Unknown delivery is sampled once for information, but never reported as verified.
- const spec = status === "unknown" ? { ...follow.expect, timeout_ms: 0, stable: 1 } : follow.expect;
- this.retireWindow(window);
- successor = this.remember({ window_ref: window }, await session.waitFor(window, spec, signal));
- const verification = successor.details.verification;
- const verdict = verification?.verdict ?? "unknown";
- postcondition = { status: status === "unknown" ? "unknown" : verdict === "satisfied" ? (preexisting ? "preexisting" : "verified") : verdict === "unsatisfied" ? "failed" : "unknown",
- ...(status === "unknown" ? { reason: "The action outcome is unknown, so its postcondition cannot be attributed to it." } : {}),
- ...(verification ? { verification } : {}) };
- } else if (follow.expect) {
- postcondition = { status: preexisting ? "preexisting" : "failed", reason: "The action was refused; nothing was sent." };
- } else if (status !== "refused") {
+ if (status !== "refused") {
if (result.details.dom) successor = result;
else ({ successor, diff: settled, crops } = await this.settle(window, session, before, follow.frame, signal));
}
@@ -1315,7 +1136,6 @@ export class ComputerResources {
: settled ?? (before.result.details.ui && after?.ui ? diffUi(before.result.details.ui, after.ui) : undefined);
// The successor replaces the action's own observation as the current one; a wait retired the earlier snapshot.
const details = { ...result.details, ...(after?.observation_id ? { observation_id: after.observation_id, ...(after.dom ? { dom: after.dom } : {}) } : {}),
- ...(postcondition ? { postcondition } : {}),
successor: after?.observation_id ? { observation_id: after.observation_id, ...(diff ? { diff } : { reason: "The input observation had no comparable UI or DOM." }) }
: { reason: status === "refused" ? "The action was refused; the input observation is unchanged but consumed." : "No successor observation could be taken." } };
return { ...result, content: [...result.content, ...crops], details };
@@ -1430,7 +1250,6 @@ export class ComputerResources {
window_ref: window, observation_id: observation.details.observation_id };
// The walk that resolved the target is the state before the action: the change list compares with it.
const before: Snapshot = { target: { window_ref: window }, result: observation };
- if (follow?.expect && observation.details.ui) follow.preexisting = combine(evaluateNative(observation.details.ui, follow.expect.conditions)) === "satisfied";
if (follow) follow = await this.withFrame(follow, window, observation.details.observation_id, cwd);
const finish = (result: ComputerResult) => follow ? this.followUp(window, before, result, follow, cwd, signal) : Promise.resolve(result);
this.snapshots.delete(observation.details.observation_id!);
@@ -1444,117 +1263,6 @@ export class ComputerResources {
: done("unknown", `${prefix.text} The click was delivered, but the element is ${state === null ? "no longer readable" : state === "mixed" ? "mixed" : state ? "checked" : "unchecked"} afterwards. Read the window before deciding; do not replay blindly.`));
}
- /**
- * Run up to 16 known steps in one call and stop at the first step that is not delivered or satisfied.
- * Browser steps are Playwright actions and conditions. Native steps resolve `target` (a Playwright locator or
- * JSON target) in fresh UI walks with auto-waiting; only the first step may use a hint or pixels of the current observation.
- */
- private async batch(address: Extract<ComputerAddress, { kind: "observation" }>, snapshot: Snapshot, command: Record<string, unknown>, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {
- only(command, ["action", "steps"]);
- const window = snapshot.target.window_ref!;
- if (address.hint !== undefined || address.delivery !== undefined || address.delayMs !== undefined) throw new ComputerRequestError("A batch takes its targets and delivery from its steps, not the address.");
- if (!Array.isArray(command.steps) || command.steps.length < 1 || command.steps.length > BATCH_STEPS) throw new ComputerRequestError(`batch needs steps: 1..${BATCH_STEPS} step objects.`);
- const session = await this.sessionFor(cwd, { window_ref: window });
- const browser = snapshot.result.details.dom !== undefined;
- type Step = { kind: "wait"; spec: WaitSpec; action: string } | { kind: "dom"; action: WebAction } | { kind: "native"; command: Record<string, unknown>; target?: ParsedSelector; describe?: string; timeout?: number };
- const steps: Step[] = command.steps.map((raw, index): Step => {
- const step = object(raw);
- const label = `steps[${index}]`;
- try {
- if (step.action === "wait_for" || step.action === "verify") return { kind: "wait", spec: session.waitSpec(window, step), action: String(step.action) };
- if (browser) return { kind: "dom", action: validateWebAction(step) };
- if (typeof step.action === "string" && LOCATOR_VERBS[step.action]) step.action = LOCATOR_VERBS[step.action];
- if (!NATIVE_BATCH_ACTIONS.includes(String(step.action))) throw Error(`action must be one of ${NATIVE_BATCH_ACTIONS.join(", ")}, wait_for or verify.`);
- if (["window_ref", "observation_id", "sequence"].some(key => key in step)) throw Error("identity fields and nested sequences are not allowed.");
- const pinned = ["hint", "element_token", "x", "y"].some(key => key in step);
- if (pinned && index > 0) throw Error("only the first step may use a hint, token or pixels; later steps use a target (a Playwright locator or JSON target).");
- let target: ParsedSelector | undefined;
- if (step.target !== undefined) {
- if (pinned) throw Error("choose target or a hint/pixel target, not both.");
- target = nativeSelector(step.target);
- }
- const timeout = step.timeout_ms;
- if (timeout !== undefined && (!Number.isSafeInteger(timeout) || (timeout as number) < 0 || (timeout as number) > 60_000)) throw Error("timeout_ms must be an integer 0..60000.");
- const { target: raw, timeout_ms: _timeout, ...nativeStep } = step;
- return { kind: "native", command: nativeStep, ...(target ? { target, describe: typeof raw === "string" ? raw : JSON.stringify(raw), timeout: (timeout as number | undefined) ?? 5_000 } : {}) };
- } catch (error) { throw new ComputerRequestError(`${label}: ${error instanceof Error ? error.message : String(error)}`); }
- });
- this.snapshots.delete(address.ref);
- const started = performance.now();
- const rows: NonNullable<ComputerResult["details"]["batch"]>["steps"] = [];
- // `issued` holds the latest result whose observation is still current; a native action consumes it.
- let issued: ComputerResult | undefined, observation = address.ref, last: ComputerResult = snapshot.result, stop = "";
- // The state the latest action started from, so the next walk or page snapshot can say what that action changed.
- let previousUi = snapshot.result.details.ui, previousDom = snapshot.result.details.dom, pendingRow: (typeof rows)[number] | undefined;
- const count = (diff: StateDiff) => diff.counts;
- const seen = (ui: UiResource | undefined) => {
- if (pendingRow && previousUi && ui) pendingRow.changes = count(diffUi(previousUi, ui));
- pendingRow = undefined;
- };
- for (const [index, step] of steps.entries()) {
- if (signal?.aborted) { stop = "Cancelled before the next step."; break; }
- if (step.kind === "wait") {
- this.retireWindow(window);
- last = this.remember({ window_ref: window }, await session.waitFor(window, step.spec, signal));
- seen(last.details.ui);
- issued = last.details.observation_id ? last : undefined;
- const verdict = last.details.verification?.verdict ?? "unknown";
- rows.push({ index, action: step.action, outcome: last.details.status, verdict });
- observation = last.details.observation_id ?? observation;
- if (last.details.status !== "observed" || verdict !== "satisfied") { stop = `Condition ${verdict} at step ${index}.`; break; }
- continue;
- }
- if (step.kind === "dom") {
- last = await session.actDom(window, observation, step.action, signal);
- issued = undefined;
- if (last.details.dom) { this.remember({ window_ref: window }, last); observation = last.details.observation_id!; issued = last; }
- } else {
- let element: string | undefined;
- if (index > 0 || step.target) {
- // Never reuse a stale observation: a target resolves in fresh UI walks; other steps take one fresh walk.
- if (step.target) {
- const action = String(step.command.action), pointer = ["click", "double_click", "right_click"].includes(action);
- const resolved = await this.resolveNative(window, session, step.target, { timeout_ms: step.timeout!, enabled: true, stable: pointer, visible: pointer, describe: step.describe! }, signal);
- if ("refused" in resolved) {
- if (resolved.observation) last = resolved.observation;
- rows.push({ index, action, outcome: "refused", reason: resolved.refused.slice(0, 600) });
- stop = `Target not resolved at step ${index}; nothing was sent for it.`; break;
- }
- issued = resolved.observation; observation = resolved.observation.details.observation_id!;
- element = resolved.element.element_token;
- seen(resolved.observation.details.ui); previousUi = resolved.observation.details.ui;
- } else {
- this.retireWindow(window);
- const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));
- if (fresh.details.status !== "observed" || !fresh.details.ui) { last = fresh; rows.push({ index, action: String(step.command.action), outcome: "refused", reason: "No current UI sample." }); stop = `No current UI before step ${index}.`; break; }
- issued = fresh; observation = fresh.details.observation_id!;
- seen(fresh.details.ui); previousUi = fresh.details.ui;
- }
- }
- this.snapshots.delete(observation);
- issued = undefined;
- last = await session.act({ ...step.command, ...(element ? { element_token: element } : {}), window_ref: window, observation_id: observation } as Action, signal);
- }
- const outcome = last.details.status;
- const row: (typeof rows)[number] = { index, action: step.kind === "dom" ? step.action.action : String(step.command.action), outcome,
- ...(outcome !== "delivered" ? { reason: computerResultText(last).split("\n{", 1)[0].slice(0, 300) } : {}) };
- // A page action returns its next snapshot; a native action's effect shows in the next walk.
- if (step.kind === "dom") { if (previousDom && last.details.dom) row.changes = count(diffDom(previousDom, last.details.dom)); previousDom = last.details.dom ?? previousDom; }
- else if (outcome === "delivered") pendingRow = row;
- rows.push(row);
- if (outcome !== "delivered") { stop = `Step ${index} ${outcome}; later steps were not sent.`; break; }
- }
- const completed = rows.filter(row => row.outcome === "delivered" || row.verdict === "satisfied").length;
- const failed = rows.at(-1)?.outcome;
- // A stop at a condition leaves every sent action delivered; computerEffect marks a refusal after delivered steps as unknown.
- const status: ComputerResult["details"]["status"] = failed === "unknown" || failed === "refused" ? failed : "delivered";
- const message = `Batch ${stop ? "stopped" : "completed"}: ${completed}/${steps.length} steps in ${Math.round(performance.now() - started)} ms. ${stop || "Every action was delivered and every condition held."} Never replay completed or uncertain steps.`;
- // The current observation is whatever the last step left: a successor snapshot or verification sample.
- const base = issued ? issued.details : { ...last.details, dom: undefined, ui: undefined, observation_id: undefined, verification: undefined };
- const details = { ...base, status, ...(last.details.window_ref ? { window_ref: last.details.window_ref } : {}),
- batch: { total: steps.length, completed, stop_reason: stop, elapsed_ms: Math.round(performance.now() - started), steps: rows } };
- return { content: [{ type: "text", text: message }], details, ...(status === "delivered" ? {} : { isError: true }) };
- }
}
/** Parse a tool-owned path without allowing an unknown computer URI to fall through to files. */
diff --git a/src/computer-tools.ts b/src/computer-tools.ts
index 6069c49..e2998a5 100644
--- a/src/computer-tools.ts
+++ b/src/computer-tools.ts
@@ -51,7 +51,7 @@ const openParameters = Type.Object({
}, { additionalProperties: false, description: "YOLO: set a WebView2 app's debugging port for its next start on that machine, so its window can be attached; opens nothing. Windows may ask the person there to approve the change." })),
}, { additionalProperties: false });
-/** The Insert command that opens what computer_open asks for at this address. */
+/** The command that opens what computer_open asks for at this address. */
export function openCommand(address: ComputerAddress, params: Static<typeof openParameters>): Record<string, unknown> {
const { app, path, args, debugging, browser, url, profile, sandbox, isolated_session, webview2_debugging } = params;
const chosen = [app !== undefined && "app", path !== undefined && "path", (browser ?? url ?? profile) !== undefined && "browser",
@@ -134,7 +134,7 @@ const waitParameters = Type.Object({
stable: Type.Optional(Type.Integer({ minimum: 1, maximum: 10, description: "wait: consecutive satisfied samples required, default 1." })),
}, { additionalProperties: false });
-/** The Insert wait_for or verify command of computer_wait. */
+/** The wait_for or verify command of computer_wait. */
export function waitCommand(params: Static<typeof waitParameters>): Record<string, unknown> {
const { window: _window, mode, conditions, ...timing } = params;
if (mode === "check" && Object.keys(timing).length) throw new ComputerRequestError(`check takes one sample, so ${Object.keys(timing).join(", ")} do not apply.`);
@@ -472,7 +472,7 @@ export function timelinePlan(params: Static<typeof timelineParameters>): InputPl
return { label: "timeline", ...shared(params), native: { action: "timeline", events: params.events }, web };
}
/**
- * Register the typed computer_* tools that replace Insert commands one cluster at a time. They are deferred: a
+ * Register the typed computer_* tools, one per operation cluster. They are deferred: a
* tool_search for computer loads them, so sessions that never use the computer pay nothing for them.
*/
export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResources): void {
@@ -494,7 +494,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
const address = resources.address(params.at);
command = openCommand(address, params);
return address;
- }, address => resources.perform(address, ctx.cwd, JSON.stringify(command), "insert", signal)));
+ }, address => resources.perform(address, ctx.cwd, JSON.stringify(command), signal)));
},
});
pi.registerTool({
@@ -524,7 +524,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
const address = resources.address(params.window);
if (address.kind !== "window") throw new ComputerRequestError("computer_wait needs a window address from Search, Read or a name.");
return address;
- }, address => resources.perform(address, ctx.cwd, JSON.stringify(command), "insert", signal)));
+ }, address => resources.perform(address, ctx.cwd, JSON.stringify(command), signal)));
},
});
pi.registerTool({
@@ -550,7 +550,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
return actionResult(await computerAction(resources, params.machine, params.action, () => {
command = filesCommand(params);
return machineAddress(resources, params.machine);
- }, address => resources.perform(address, ctx.cwd, JSON.stringify(command), "insert", signal)));
+ }, address => resources.perform(address, ctx.cwd, JSON.stringify(command), signal)));
},
});
pi.registerTool({
@@ -564,7 +564,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
return actionResult(await computerAction(resources, params.machine, `clipboard_${params.action}`, () => {
command = clipboardCommand(params);
return machineAddress(resources, params.machine);
- }, address => resources.perform(address, ctx.cwd, JSON.stringify(command), "insert", signal)));
+ }, address => resources.perform(address, ctx.cwd, JSON.stringify(command), signal)));
},
});
pi.registerTool({
@@ -584,15 +584,15 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
}
return machineAddress(resources, params.at);
}, async address => "views" in run
- ? (await resources.read(address, ctx.cwd, run.views, signal))!
- : resources.perform(address, ctx.cwd, JSON.stringify(run.command), "insert", signal)));
+ ? (await resources.sequence(address, ctx.cwd, run.views, signal))!
+ : resources.perform(address, ctx.cwd, JSON.stringify(run.command), signal)));
},
});
/** Run one typed input on a window: the plan is built and checked before anything reaches the machine. */
- const input = (window: string, action: string, build: () => InputPlan, cwd: string, signal?: AbortSignal) => {
+ const input = async (window: string, action: string, build: () => InputPlan, cwd: string, signal?: AbortSignal) => {
let plan: InputPlan | undefined;
- return computerAction(resources, window, action, () => { plan = build(); return resources.address(window); },
- address => resources.input(address, cwd, plan!, signal)).then(actionResult);
+ return actionResult(await computerAction(resources, window, action, () => { plan = build(); return resources.address(window); },
+ address => resources.input(address, cwd, plan!, signal)));
};
const aimed = (args: Record<string, unknown>) => args.target !== undefined ? String(args.target) : args.x !== undefined ? `${args.x},${args.y}` : "";
const inputAnnotations = { readOnlyHint: false, destructiveHint: true, openWorldHint: true };
@@ -660,7 +660,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
...rendered("computer_display", args => `${args.display} ${args.width}x${args.height}`),
async execute(_id, params, signal, _update, ctx) {
return actionResult(await computerAction(resources, params.display, "resize", () => resources.address(params.display),
- address => resources.perform(address, ctx.cwd, JSON.stringify({ action: "resize", width: params.width, height: params.height }), "insert", signal)));
+ address => resources.perform(address, ctx.cwd, JSON.stringify({ action: "resize", width: params.width, height: params.height }), signal)));
},
});
}
diff --git a/src/isolated-session.ts b/src/isolated-session.ts
index 64737e2..5e7c0c0 100644
--- a/src/isolated-session.ts
+++ b/src/isolated-session.ts
@@ -66,7 +66,7 @@ export class IsolatedSession {
const local = (await this.ps("$env:LOCALAPPDATA", signal)).trim();
this.wxc = `${local}\\PiComputerUse\\mxc\\wxc-exec.exe`;
if ((await this.ps(`Test-Path -LiteralPath ${psString(this.wxc)}`, signal)).trim() !== "True")
- throw new ComputerRequestError(`MXC is not installed on this machine (${this.wxc}). Ask the user before installing it; with their permission Insert {"action":"setup","part":"mxc"} into this machine's address, then open the session again.`);
+ throw new ComputerRequestError(`MXC is not installed on this machine (${this.wxc}). Ask the user to install it with /computer-setup mxc, then open the session again.`);
const windows = `${local}\\PiComputerUse\\sessions\\${this.ref}`;
this.workdir = { windows, unix: await this.host.path(`${local}\\PiComputerUse\\sessions`, "unix").then(parent => `${parent}/${this.ref}`) };
await this.host.mkdir(this.workdir.unix);
@@ -127,6 +127,6 @@ export class IsolatedSession {
describe(source: string): string {
return [`# Isolated Windows session`, `Address: ${source}`, `Agent user: ${this.user ?? "unknown"}`,
"A separate Windows session for the agent on this machine (MXC isolation_session): its own desktop, clipboard and input, invisible to the human. Several can run at once.",
- "Search this address for its windows. Insert {\"action\":\"open\",\"app\":\"...\"} to start an app in it, or {\"action\":\"close\"} to remove it."].join("\n");
+ "Search this address for its windows. computer_open at this address starts an app in it; Delete removes it."].join("\n");
}
}
diff --git a/src/machine.ts b/src/machine.ts
index a82ce3c..88878af 100644
--- a/src/machine.ts
+++ b/src/machine.ts
@@ -76,7 +76,7 @@ export function sandboxFactory(options: { settings: () => CuaVisionSettings; ret
export function machineFactory(options: { getMode: () => ComputerMode; settings: () => CuaVisionSettings; retainRaw?: (image: Content) => void }): MachineFactory {
return async (cwd, name, signal) => {
const machine = await machineConfig(cwd, name);
- const refresh = `setup part machine (user: /computer-setup machine ${machine.ssh} ${name}; agent, after asking: Insert {"action":"setup","part":"machine","ssh":"${machine.ssh}","name":"${name}"} into computer:machine/local)`;
+ const refresh = `setup part machine (user: /computer-setup machine ${machine.ssh} ${name})`;
const runner = sshRunner(machine.ssh);
const driver = new McpDriver({ command: machine.command, ...(machine.runtime ? { runtime: machine.runtime } : {}), ssh: machine.ssh });
try { await waitForGuestDriver(driver, signal, 30_000); }
diff --git a/src/provider.ts b/src/provider.ts
index e516cdd..00b6aa5 100644
--- a/src/provider.ts
+++ b/src/provider.ts
@@ -19,11 +19,28 @@ const record = (value: unknown): Record<string, unknown> => value !== null && ty
const label = (value: unknown) => typeof value === "string" ? value.replace(/[\r\n\t]/g, " ").slice(0, 1024) : "";
/** Agent guidance shared by the IDE plugin and the standalone tools. */
-export const COMPUTER_PATH_HELP = "Read docs:computer for native resources. computer:session, computer:machine/<name> (local, or a configured machine) with its /windows, /apps and /displays, its /display/primary or listed /display/ref, and returned computer:window/ref, observation/id, window-state/id or sequence/id, each running on the machine that issued it; results show short forms such as computer:<machine>/w3qk (o observation, s sequence), and computer:<name> after Insert {\"action\":\"name\",\"as\":\"<name>\"} into a window. Read window overviews before input; display/detail/frames are readonly.";
-export const COMPUTER_VIEWS_HELP = "Computer: image/ui/full; dom alone or dom:{target,boxes,depth,log} on a browser window opened by open_browser or attached with attach_browser returns a fresh Playwright aria snapshot with [ref=...] values of the active tab; log:true adds the retained console/page-error/network history; target is {ref} or locator fields role,name,label,text,placeholder,test_id,css,exact,frames. On a DOM observation: image is a page screenshot, element:crop|highlight,<ref> an element image; Search filters its lines. state alone on a window returns fresh restore or management metadata without pixels/UI, plus whether the window is hung and its process memory/CPU. ocr (alone or with capture options on a window, alone on an observation, or with region:x,y,width,height on an observation) adds Windows OCR lines with boxes in overview pixels; use maxLongEdge 3840 for small text. aria (alone or with ui capture options, on a window) returns its UI as a Playwright aria snapshot with hints as [ref=...]. changes (alone, or with ui, image or full and capture options, on a window) takes a fresh capture and lists only rows changed, added or removed since that window's previous observation, with the new observation's hints, plus changed pixel areas with before/after crops instead of the full image. regions (alone or with image/full options on a window, alone on its observation) lists icons and text areas found by Cua perception with boxes in overview pixels; click a box center with pixel input. capture:{maxLongEdge,scale,max_elements,max_depth,hints,guides}. region:x,y,width,height or element:inspect|scope|highlight|crop|text,hint on an observation; element:text returns one element's full text. sequence:{duration_ms<=60000,fps<=30,max_frames<=300,timeout_ms,maxLongEdge,scale,region} on window/display reports actual_fps and last_frame_age_ms; frames:first|last|all|endpoints|changed or frames:{indexes:[0]} / {from:0,to:2} / {select:'changed'} on sequence; add as:'sheet' (one tiled image with index/time labels, optional columns) or as:'max'|'min' (one per-pixel composite), with optional region:{x,y,width,height} in frame pixels and maxLongEdge. Detail/frames are readonly; text offset/limit select UI lines, never image cells.";
+export const COMPUTER_PATH_HELP = "Read docs:computer for native resources. computer:session, computer:machine/<name> (local, or a configured machine) with its /windows, /apps and /displays, its /display/primary or listed /display/ref, and returned computer:window/ref, observation/id or sequence/id, each running on the machine that issued it; results show short forms such as computer:<machine>/w3qk (o observation, s sequence), and computer:<name> after computer_window name. Read only looks; the computer_* tools act.";
+export const COMPUTER_VIEWS_HELP = "Computer: image/ui/full; dom alone or dom:{target,boxes,depth,log} on a browser window opened or attached with computer_open returns a fresh Playwright aria snapshot with [ref=...] values of the active tab; log:true adds the retained console/page-error/network history; target is {ref} or locator fields role,name,label,text,placeholder,test_id,css,exact,frames. On a DOM observation: image is a page screenshot, element:crop|highlight,<ref> an element image; Search filters its lines. state alone on a window returns its minimized, maximized, visible and foreground state and bounds without pixels/UI, plus whether the window is hung and its process memory/CPU; computer_window changes them. ocr (alone or with capture options on a window, alone on an observation, or with region:x,y,width,height on an observation) adds Windows OCR lines with boxes in overview pixels; use maxLongEdge 3840 for small text. aria (alone or with ui capture options, on a window) returns its UI as a Playwright aria snapshot with hints as [ref=...]. changes (alone, or with ui, image or full and capture options, on a window) takes a fresh capture and lists only rows changed, added or removed since that window's previous observation, with the new observation's hints, plus changed pixel areas with before/after crops instead of the full image. regions (alone or with image/full options on a window, alone on its observation) lists icons and text areas found by Cua perception with boxes in overview pixels; click a box centre with computer_click at its x,y. capture:{maxLongEdge,scale,max_elements,max_depth,hints,guides}. region:x,y,width,height or element:inspect|scope|highlight|crop|text,hint on an observation; element:text returns one element's full text. frames:first|last|all|endpoints|changed or frames:{indexes:[0]} / {from:0,to:2} / {select:'changed'} on a sequence address from computer_record; add as:'sheet' (one tiled image with index/time labels, optional columns) or as:'max'|'min' (one per-pixel composite), with optional region:{x,y,width,height} in frame pixels and maxLongEdge. Detail/frames are readonly; text offset/limit select UI lines, never image cells.";
export const COMPUTER_SEARCH_HELP = "Search computer:machine/<name>/apps or computer:machine/<name>/windows (local is this machine) for installed-app/permitted-window metadata, or an observation for its collected UI/readonly DOM. Read computer:machine/<name>/displays for monitor metadata. Discovery launches nothing; observation Search performs no new native walk or DOM query. Treat missing matches in a partial projection as unknown.";
export const COMPUTER_DELETE_HELP = "A computer window address closes that window like its close button (add ?force=true to kill the hung app of a window this session opened); a sandbox, isolated session or machine address releases it; computer:session releases everything computer use started.";
-export const COMPUTER_INPUT_HELP = "Computer Write sends content literally through type_text to a fresh UI/image observation address. On a DOM observation of a browser opened with Insert {\"action\":\"open_browser\"} (YOLO: \"profile\":\"agent\" keeps the agent's own logins between sessions), or of a running Chromium-based browser attached in YOLO with Insert {\"action\":\"attach_browser\"} into its window address (Chrome or Edge through a debugging port, the chrome://inspect/#remote-debugging switch or the Playwright Extension; an Electron/CEF app started with --remote-debugging-port=0 in open's args; a WebView2 app started with open \"debugging\":true or after {\"action\":\"webview2_debugging\",\"app\":\"x.exe\",\"enabled\":true} on its machine; VS Code's integrated browser through the Pi Browser Bridge; closing the session detaches), Insert one Playwright action such as {\"action\":\"fill\",\"target\":{\"label\":\"Name\"},\"text\":\"value\"}; actions target the active tab, and tab/navigation/dialog actions are listed in docs:computer. Insert {\"action\":\"open_sandbox\"} into a machine address to work in a separate disposable Windows Sandbox that does not touch the human's desktop, or {\"action\":\"open_session\"} for an isolated agent session on the same machine (several can run at once, each with its own desktop and foreground input); use the returned computer:sandbox/... address for Search, open and close (docs:computer). Machines are computer:machine/local (this one) and computer:machine/<configured name> (another Windows machine over SSH, started on first use); each takes the same commands. Insert {\"action\":\"put\",\"from\":\"<workspace file>\"}, {\"action\":\"get\",\"name\":\"<file>\",\"to\":\"<workspace path>\"} or {\"action\":\"files\"} into a machine address to move files through the session's own folder on that machine (removed on close); point app Open/Save dialogs there. The same addresses take {\"action\":\"clipboard_write\",\"text\"|\"image\":...}, {\"action\":\"clipboard_read\",\"image\":true}, {\"action\":\"record_start\",\"seconds\":30} and {\"action\":\"record_stop\",\"to\":\"<workspace path>\"} (main-display video) and {\"action\":\"audio_record\",\"seconds\":10,\"transcribe\":true} (playback audio with a transcript) in YOLO. Insert {\"action\":\"timeline\",\"events\":[{at_ms,type,...}]} with delivery=foreground into an observation for timed key/button/move/move_relative/wheel/pen events; everything held is released at the end (docs:computer). Insert {\"action\":\"resize\",\"width\":1920,\"height\":1080} into a display address to change that display's resolution in YOLO; closing the session restores it. Only after the user explicitly allows it, Insert {\"action\":\"setup\",\"part\":\"browser|firefox|perception|mxc|vscode-bridge\"} into a machine address to install a missing optional part there (docs:computer). Insert {\"action\":\"menu\",\"path\":[\"File\",\"Save As...\"]} into an observation to invoke a menu item by its labels, and {\"action\":\"kill\"} into the window address of an app this session opened to force-close it. Insert an action with a Playwright locator target, such as {\"action\":\"click\",\"target\":\"getByRole('button', { name: 'Save' })\"} or fill/check/select, into a native window address to act without a prior Read; it waits until exactly one element is ready. Insert {\"action\":\"wait_for\"|\"verify\",\"conditions\":[...]} into a window address to check UI state with the same locators, count or an aria snapshot template; every action reports what it changed in the window: UI rows, and changed pixel areas with before/after crops after an action on an image or full observation (\"successor\":false skips it), and \"expect\" adds a postcondition; or Insert {\"action\":\"batch\",\"steps\":[...]} into an observation for a fail-stop chain (docs:computer). Use no Write or native input on a DOM observation. Use optional ?hint=...&delivery=foreground&delay_ms=0 only on input observations. No implicit select-all, submit or program execution. Computer Insert sends {\"action\":\"restore\"} to a fresh window-state address for explicit restoration and activation without keys/clicks, or one JSON action to an observation, or {\"action\":\"open\",\"app\":\"Exact installed name\"} (YOLO also {\"action\":\"open\",\"path\":\"C:\\\\...\\\\app.exe\",\"args\":[...]}) to a machine address, and {\"action\":\"close\"} to a machine address or to computer:session for every machine (the result lists each released part). Close releases runtime, not documents. Omit window_ref/observation_id in JSON. Input consumes observations even on refusal; the action's successor observation is current for the next input. With \"successor\":false, delivered pixel-only input keeps its observation until the window is resized; otherwise refresh with Read. One sequence takes up to 2000 key, click and stroke steps. Never replay unknown prefixes. Foreground input requires an observed background refusal and pauses for 10 s after the human's own keyboard or mouse input; restore is a separate explicit activation, not input delivery. Safe/YOLO is user-controlled.";
+/** Each Insert command the typed tools replaced, by the tool that does it now. */
+const REPLACED: [RegExp, string][] = [
+ [/^(click|double_click|right_click|invoke|check|uncheck)$/, "computer_click"], [/^(type_text|type|fill|set_value)$/, "computer_type"],
+ [/^(key|hotkey|press)$/, "computer_press"], [/^(scroll|wheel)$/, "computer_scroll"], [/^(drag|hold)$/, "computer_drag"], [/^(hover|move_pointer)$/, "computer_hover"],
+ [/^(select|select_option|menu)$/, "computer_select"], [/^(timeline|sequence)$/, "computer_timeline"],
+ [/^(open|open_browser|open_sandbox|open_session|attach_browser|webview2_debugging)$/, "computer_open"],
+ [/^(move|resize|minimize|maximize|focus|restore|name)$/, "computer_window (resize of a display: computer_display)"], [/^(wait_for|verify)$/, "computer_wait"],
+ [/^(navigate|back|forward|reload|wait_load|new_tab|switch_tab|close_tab|dialog|upload|wait_download|cancel_download|evaluate)$/, "computer_browser"],
+ [/^(put|get|files)$/, "computer_files"], [/^clipboard_/, "computer_clipboard"], [/^(record_start|record_stop|audio_record)$/, "computer_record"],
+ [/^(close|kill)$/, "Delete (kill: Delete with ?force=true)"], [/^setup$/, "/computer-setup, which only the user runs (ask them)"], [/^batch$/, "a Codemode script"],
+];
+/** Why Write or Insert refuses a computer address, naming the tool that does what it was sent. */
+export function computerEditRefusal(tool: "write" | "insert", input: Record<string, unknown>): string {
+ let action: unknown = tool === "write" ? "type_text" : undefined;
+ if (tool === "insert" && typeof input.text === "string") try { action = (JSON.parse(input.text) as { action?: unknown })?.action; } catch { /* no action to map */ }
+ const replacement = typeof action === "string" ? REPLACED.find(([pattern]) => pattern.test(action as string))?.[1] : undefined;
+ return `${tool === "write" ? "Write" : "Insert"} does not act on computer resources; nothing was sent. ${replacement ? `Use ${replacement}. ` : ""}Each operation has its own computer_* tool: tool_search loads them with "computer open window", "computer click hover drag scroll", "computer type press select timeline", "computer wait browser" or "computer files clipboard record".`;
+}
/** Present native resources without dumping provenance; full receipts remain in structured data. */
export function presentComputerResult(result: ComputerResult, full = false): string {
@@ -33,13 +50,11 @@ export function presentComputerResult(result: ComputerResult, full = false): str
if (d.window_ref) lines.push(`Window: ${computerSource("window", d.window_ref)}`);
if (d.window_state_id && d.status === "observed") {
const sample = d.window_state_kind === "management" ? record(state.state) : state;
- lines.push(`State: ${computerSource("window-state", d.window_state_id)}`,
- `Window state: minimized=${sample.minimized}, visible=${sample.visible}; foreground=${sample.foreground_at_target}, keyboard_focus=${sample.keyboard_focus_at_target}${d.window_state_kind === "management" ? `; maximized=${sample.maximized}` : ""}`);
+ lines.push(`Window state: minimized=${sample.minimized}, visible=${sample.visible}; foreground=${sample.foreground_at_target}, keyboard_focus=${sample.keyboard_focus_at_target}${d.window_state_kind === "management" ? `; maximized=${sample.maximized}` : ""}`);
if (d.window_state_kind === "management") {
lines.push(`Bounds: ${JSON.stringify(sample.bounds)}; source=${sample.bounds_source}; unit=${sample.coordinate_space}`,
- `Current virtual desktop: ${record(sample.virtual_desktop).on_current_desktop}`,
- "Use one Insert move/resize/minimize/maximize/focus or restore. Frame units are desktop pixels, not pointer coordinates.");
- } else lines.push('Insert {"action":"restore"} once explicitly restores and activates this window. No pixel, typing or pointer authority.');
+ `Current virtual desktop: ${record(sample.virtual_desktop).on_current_desktop}`);
+ }
}
const invalidManagement = state.code === "tool_output_invalid" ? record(state.invalid_output) : {};
const management = typeof state.mutation_attempted === "boolean" ? state : invalidManagement;
@@ -56,7 +71,7 @@ export function presentComputerResult(result: ComputerResult, full = false): str
`Measured state: visible=${state.visible}; foreground=${state.foreground_at_target}, keyboard_focus=${state.keyboard_focus_at_target}`);
}
const id = d.observation_id ?? d.agent_image?.image_id;
- if (typeof id === "string" && (d.status === "observed" || (d.dom || d.batch) && d.status === "delivered")) lines.push(`Observation: ${computerSource("observation", id)}`);
+ if (typeof id === "string" && (d.status === "observed" || d.dom && d.status === "delivered")) lines.push(`Observation: ${computerSource("observation", id)}`);
if (d.agent_image) {
const image = d.agent_image, output = record(image.output), configuration = record(image.configuration);
lines.push(`Image: ${output.width}x${output.height} ${label(output.mimeType)}; ${label(image.route)}; ${label(image.coordinate_space)}`);
@@ -89,11 +104,11 @@ export function presentComputerResult(result: ComputerResult, full = false): str
if (message) lines.push(message);
}
const dom = d.dom;
- lines.push(dom.active ? `Page: ${label(dom.title)} | ${label(dom.url)} | tab ${dom.active}` : "Page: none; no tab is active. Insert {\"action\":\"switch_tab\",\"tab\":\"<id>\"}.");
+ lines.push(dom.active ? `Page: ${label(dom.title)} | ${label(dom.url)} | tab ${dom.active}` : "Page: none; no tab is active. Use computer_browser switch_tab.");
for (const tab of dom.tabs ?? [])
lines.push(`Tab ${tab.id}${tab.active ? " (active)" : ""}: ${label(tab.title)} | ${label(tab.url)} | ${tab.load}${tab.opener ? ` | opened by ${tab.opener}` : ""}${tab.dialog ? ` | ${tab.dialog} dialog open` : ""}`);
if (dom.dialog) lines.push(`Dialog: ${dom.dialog.type} ${JSON.stringify(dom.dialog.message)}${dom.dialog.default_value !== undefined ? ` (default ${JSON.stringify(dom.dialog.default_value)})` : ""}. ` +
- "The page waits for it: Insert {\"action\":\"dialog\",\"accept\":true} or false; an accepted prompt takes \"text\". Other page actions refuse until then.");
+ "The page waits for it: answer with computer_browser dialog accept true or false; an accepted prompt takes text. Other page actions refuse until then.");
for (const item of dom.downloads ?? [])
lines.push(`Download ${item.id}: ${label(item.filename)} | ${item.state} | tab ${item.tab}${item.bytes !== undefined ? ` | ${item.bytes} bytes` : ""}${item.path ? ` | ${label(item.path)}` : ""}${item.error ? ` | ${label(item.error)}` : ""}`);
for (const event of dom.events ?? []) lines.push(`Event: ${label(event)}`);
@@ -160,13 +175,6 @@ export function presentComputerResult(result: ComputerResult, full = false): str
lines.push(`OCR: ${d.text_lines.length} lines; boxes in this observation's overview pixels. Recognised text can be wrong; confirm with pixels or UI before acting on it.`);
for (const line of d.text_lines) lines.push(`"${label(line.text)}" | x=${line.x} y=${line.y} w=${line.width} h=${line.height}`);
}
- if (d.batch) for (const step of d.batch.steps)
- lines.push(`Step ${step.index}: ${step.action} → ${step.outcome}${step.verdict ? ` (${step.verdict})` : ""}${step.changes ? `; ${changeCount(step.changes)}` : ""}${step.reason ? `; ${label(step.reason)}` : ""}`);
- if (d.postcondition) {
- const p = d.postcondition;
- lines.push(`Postcondition: ${p.status}${p.reason ? `; ${p.reason}` : ""}`);
- p.verification?.conditions.forEach((condition, i) => lines.push(` Condition ${i + 1} → ${condition.verdict}${condition.observed != null ? ` (observed ${JSON.stringify(label(condition.observed))})` : ""}${condition.reason ? `; ${label(condition.reason)}` : ""}`));
- }
if (d.successor) {
const s = d.successor, diff = s.diff;
lines.push(s.observation_id ? `Successor: ${computerSource("observation", s.observation_id)}` : `Successor: none. ${s.reason ?? ""}`);
@@ -174,12 +182,11 @@ export function presentComputerResult(result: ComputerResult, full = false): str
else if (s.reason) lines.push(s.reason);
}
if (d.selected_frames) lines.push(`Selected frames: ${d.selected_frames.map(frame => frame.index).join(", ")}; readonly, input=none`);
- lines.push(d.successor?.observation_id ? "Read the successor observation for the full state; it is current for input."
- : d.batch && d.observation_id ? "The batch left this observation current for input. Never replay completed or uncertain steps."
- : d.verification ? (d.verification.verdict === "satisfied" ? "Conditions held; this observation is current for input." : "Not verified: neither proof of success nor of failure beyond the listed verdicts. Never replay uncertain input blindly.")
- : d.dom ? "Insert one JSON action into this observation, e.g. {\"action\":\"click\",\"target\":{\"ref\":\"e2\"}}. Read element:crop|highlight,<ref> or image for page pixels. Refs belong to the latest snapshot."
- : d.window_state_id ? "Use the still-current state once. Cached Read does not renew authority; state does not grant typing or pointer input."
- : d.status === "observed" ? "Detail/display/sequence frames are readonly; fresh window Read is required after detail or input."
+ lines.push(d.successor?.observation_id ? "Refs in this change list are current for the next input; Read the window for its full state."
+ : d.verification ? (d.verification.verdict === "satisfied" ? "Conditions held; refs of this sample are current for input." : "Not verified: neither proof of success nor of failure beyond the listed verdicts. Never replay uncertain input blindly.")
+ : d.dom ? "Act on the window with the computer_* input tools, a ref such as e2 as target. Read element:crop|highlight,<ref> or image for page pixels. Refs belong to the latest snapshot."
+ : d.window_state_id ? "This Read only looks; computer_window changes the window's state."
+ : d.status === "observed" ? "Refs of this Read are current for the next input to its window; detail, display and frame Reads give none."
: "Native delivery is not verified application state. Never replay unknown input; Read a fresh window overview/state before another action.");
const text = lines.join("\n");
if (full || !d.dom || Buffer.byteLength(text) <= 24_000) return text;
@@ -189,9 +196,6 @@ export function presentComputerResult(result: ComputerResult, full = false): str
return `${preview.join("\n")}\nDOM text preview reached its byte limit after ${preview.length} of ${lines.length} lines. Read this observation with offset and limit to page through it (each page names the next offset), or Search it; omitted lines are not absence.`;
}
-/** One step's effect in a batch row: counts, or that nothing changed. */
-const changeCount = (c: { changed: number; added: number; removed: number }) =>
- c.changed + c.added + c.removed ? `${c.changed} changed, ${c.added} added, ${c.removed} removed` : "nothing changed";
/** Lines of a compact UI, DOM or pixel change list; changed and added rows carry the new observation's hints. */
function diffLines(diff: StateDiff, afterAction = false): string[] {
@@ -242,12 +246,12 @@ function fittingLines(lines: readonly string[], budget: number): string[] {
export function computerEffect(result: ComputerResult): "applied" | "not-applied" | "unknown" {
if (!result.isError && ["delivered", "observed", "closed"].includes(result.details.status)) return "applied";
const sequence = record(result.details.state?.sequence);
- const delivered = (typeof sequence.completed === "number" && sequence.completed > 0) || (result.details.batch?.steps.some(step => step.outcome === "delivered") ?? false);
+ const delivered = typeof sequence.completed === "number" && sequence.completed > 0;
return result.details.status === "refused" && !delivered ? "not-applied" : "unknown";
}
/**
- * The value a Codemode script receives from a computer Write or Insert. Every outcome resolves with this
+ * The value a Codemode script receives from a computer_* tool. Every outcome resolves with this
* shape, so scripts branch on `outcome` instead of catching errors. Delivery never verifies application state.
*/
export interface ComputerReceipt {
@@ -260,11 +264,11 @@ export interface ComputerReceipt {
action: string | null;
source: string;
window: string | null;
- /** The sandbox address an open_sandbox command started. */
+ /** The sandbox address computer_open started. */
sandbox?: string;
/** The new current observation issued by this call, such as a DOM action's next snapshot. */
observation: string | null;
- /** Whether the addressed observation or window state can no longer be used for input. */
+ /** Whether the addressed observation can no longer be used for input. */
input_consumed: boolean;
application_effect: "unverified";
reason: string | null;
@@ -275,19 +279,15 @@ export interface ComputerReceipt {
/** Verdict of a wait_for/verify command, with one entry per requested condition. */
verification?: { verdict: "satisfied" | "unsatisfied" | "unknown"; timed_out: boolean; samples: number; elapsed_ms: number; reason?: string;
conditions: { index: number; verdict: "satisfied" | "unsatisfied" | "unknown"; observed?: string | null; reason?: string }[] };
- /** Per-step results of a sequence or batch. */
+ /** Per-step results of a native sequence. */
steps?: { index: number; action: unknown; status: unknown; verdict?: string; reason?: string }[];
- /** Why a batch stopped before its last step. */
- stopped?: string;
- /** Declared postcondition: verified by this action, already true before it, not reached, or undecidable. */
- postcondition?: { status: "verified" | "preexisting" | "failed" | "unknown"; reason?: string; verdicts?: string[] };
/** Compact change list from the action's input observation to `observation`. */
changes?: { kind: string; changed: string[]; added: string[]; removed: string[]; counts: { changed: number; added: number; removed: number };
page?: Record<string, unknown>; note?: string };
/** The same receipt the model sees. */
text: string;
}
-/** Build the script receipt for one Write/Insert result, or for a request rejected before dispatch. */
+/** Build the script receipt for one computer action, or for a request rejected before dispatch. */
export function computerReceipt(source: string, action: string | null, outcome: { result: ComputerResult; inputConsumed: boolean; text: string }
| { error: unknown; inputConsumed: boolean }): ComputerReceipt {
if ("error" in outcome) {
@@ -316,10 +316,6 @@ export function computerReceipt(source: string, action: string | null, outcome:
...(d.verification ? { verification: { verdict: d.verification.verdict, timed_out: d.verification.timed_out, samples: d.verification.samples,
elapsed_ms: d.verification.elapsed_ms, ...(d.verification.reason ? { reason: d.verification.reason } : {}), conditions: d.verification.conditions } } : {}),
...(Array.isArray(sequence.steps) ? { steps: (sequence.steps as Record<string, unknown>[]).map(step => ({ index: step.index as number, action: step.action, status: step.status })) } : {}),
- ...(d.batch ? { steps: d.batch.steps.map(step => ({ index: step.index, action: step.action, status: step.outcome, ...(step.verdict ? { verdict: step.verdict } : {}), ...(step.reason ? { reason: step.reason } : {}) })),
- ...(d.batch.stop_reason ? { stopped: d.batch.stop_reason } : {}) } : {}),
- ...(d.postcondition ? { postcondition: { status: d.postcondition.status, ...(d.postcondition.reason ? { reason: d.postcondition.reason } : {}),
- ...(d.postcondition.verification ? { verdicts: d.postcondition.verification.conditions.map(condition => condition.verdict) } : {}) } } : {}),
...(d.successor?.diff ? { changes: d.successor.diff } : {}),
text: outcome.text,
};
@@ -350,18 +346,9 @@ export async function readComputer(resources: ComputerResources, source: string,
details: { source: address.source, resolvedBy: "computer", presentationTruncated: truncated, ...(result ? { computer: result.details } : {}) } };
}
-/** One computer Write or Insert. Every outcome returns a summary and a script receipt instead of throwing. */
-export async function performComputer(resources: ComputerResources, tool: "write" | "insert", source: string,
- input: Record<string, unknown>, cwd: string, signal?: AbortSignal) {
- const literal = input[tool === "write" ? "content" : "text"];
- let action: string | null = tool === "write" ? "type_text" : null;
- if (tool === "insert" && typeof literal === "string") try { const parsed = JSON.parse(literal); if (typeof parsed?.action === "string") action = parsed.action; } catch { /* reported by perform */ }
- return computerAction(resources, source, action, () => {
- const fields = tool === "write" ? ["path", "content"] : ["path", "text"];
- if (Object.keys(input).some(key => !fields.includes(key))) throw new ComputerRequestError("Computer input accepts no text anchors, ranges or insertion flags; its observation address binds the target.");
- if (typeof literal !== "string") throw new ComputerRequestError("Provide literal content for Write or one JSON command in Insert text.");
- return resources.address(source);
- }, address => resources.perform(address, cwd, literal as string, tool, signal));
+/** Write or Insert on a computer address: a refused receipt that names the computer_* tool to use instead. */
+export function refuseComputerEdit(resources: ComputerResources, tool: "write" | "insert", source: string, input: Record<string, unknown>) {
+ return computerAction(resources, source, null, () => { throw new ComputerRequestError(computerEditRefusal(tool, input)); }, () => Promise.reject(new Error("unreachable")));
}
/**
* Run one computer operation and present its outcome as an action result: the shortened summary, any images, and the
@@ -421,7 +408,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cw
const guide = await readFile(new URL("../docs/computer-resources.md", import.meta.url), "utf8");
connectAgentDocumentation(pi, [{ id: "computer", description: "Native computer resources, readonly vision and observation-bound input through ordinary IDE tools",
markdown: guide,
- triggers: ["read", "search", "write", "insert"].map(tool => ({ tool, resourcePrefixes: ["computer:"] })) }]);
+ triggers: ["read", "search"].map(tool => ({ tool, resourcePrefixes: ["computer:"] })) }]);
const resources = new ComputerResources(getSession, closeSession, openSandbox, connectMachine, getMode);
// Pi shutdown releases every machine, sandbox and session this computer use started.
pi.on("session_shutdown", () => resources.closeAll().then(() => undefined));
@@ -433,7 +420,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cw
// Delete's force option is not part of the address; Delete itself checks where it applies.
const address = resources.address(source.endsWith(FORCE) ? source.slice(0, -FORCE.length) : source);
return { kind: "resolved", resource: { source, skipPostEdit: true,
- // An expired reference still resolves, so Write/Insert reach the handler and return a refused receipt.
+ // An expired reference still resolves, so Delete reaches its handler and returns a refused receipt.
read: async () => {
try { return [{ type: "text", text: resources.shorten(resources.descriptor(address)) }]; }
catch (error) { if (error instanceof ComputerRequestError) return [{ type: "text", text: resources.shorten(`# Computer resource\nSource: ${source}\n${error.message}`) }]; throw error; }
@@ -485,16 +472,16 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cw
for (const tool of ["write", "insert", "delete"] as const) api.addToolRenderer(computerActionRenderer(tool));
renderersRegistered = true;
});
+ // Write and Insert do not act on computer addresses; they refuse instead of treating the address as a file.
for (const tool of ["write", "insert"] as const) api.tool(tool).addSemanticHandler({
matches: input => owns(record(input).path),
- execute: (context, input) => performComputer(resources, tool, context.sourceFor("path"), record(input), context.cwd, context.signal),
+ execute: (context, input) => refuseComputerEdit(resources, tool, context.sourceFor("path"), record(input)),
});
api.tool("delete").addSemanticHandler({
matches: input => owns(record(input).path),
execute: (context, input) => deleteComputer(resources, context.sourceFor("path"), record(input), context.cwd, context.signal),
});
api.tool("delete").describeField("path", COMPUTER_DELETE_HELP);
- api.describe(COMPUTER_INPUT_HELP);
} })),
]);
return { resources, guide, attached: () => accepted.size === 3 };
diff --git a/src/sandbox.ts b/src/sandbox.ts
index 1424fd0..2f4f763 100644
--- a/src/sandbox.ts
+++ b/src/sandbox.ts
@@ -212,7 +212,7 @@ export class WindowsSandbox {
const running = await this.ps(`if (Get-Process WindowsSandboxRemoteSession -ErrorAction SilentlyContinue) { 'running' }
else { Get-Process WindowsSandboxServer -ErrorAction SilentlyContinue | Stop-Process -Force }`, signal);
if (running.includes("running")) throw new ComputerRequestError("Windows Sandbox is already running on this machine, and Windows allows only one. "
- + "Close it first (Insert close into its computer:sandbox address if this session opened it), then open a new one. Nothing was started.");
+ + "Close it first (Delete its computer:sandbox address if this session opened it), then open a new one. Nothing was started.");
const startedAt = this.startedAt = new Date().toISOString();
this.launched = true;
await this.ps(`Start-Process explorer.exe -ArgumentList ${psString(`${windows}\\sandbox.wsb`)}`, signal);
@@ -298,7 +298,7 @@ ${this.wsb(`exec --id ${this.id} --run-as ExistingLogin -c 'powershell.exe -NoPr
const lines = [`# Windows Sandbox`, `Address: ${source}`, `Sandbox id: ${this.id ?? "unknown"}`,
"The sandbox window stays minimized on the host; closing it stops the sandbox. Clipboard, audio, video and printers are not shared."];
for (const share of this.shares) lines.push(`Shared: ${share.host} -> ${share.guest} (${share.writable ? "read/write" : "read-only"})`);
- lines.push("Search this address for guest windows. Insert {\"action\":\"open\",\"app\":\"...\"} to start a guest app, or {\"action\":\"close\"} to destroy the sandbox.");
+ lines.push("Search this address for guest windows. computer_open at this address starts a guest app; Delete destroys the sandbox.");
return lines.join("\n");
}
}
diff --git a/src/session.ts b/src/session.ts
index 1d1b142..f40ab25 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -113,8 +113,6 @@ export interface ComputerResult {
rendered_frames?: Omit<RenderedFrames, "png">;
/** Outcome of a bounded wait_for/verify; the conditions as requested, in order. */
verification?: WaitResult & { spec: WaitSpec };
- /** Declared postcondition of an action: verified now, already true before, not reached, or undecidable. */
- postcondition?: { status: "verified" | "preexisting" | "failed" | "unknown"; reason?: string; verification?: WaitResult & { spec: WaitSpec } };
/** The observation taken after an action and its compact difference from the action's input observation. */
successor?: { observation_id?: string; diff?: StateDiff; reason?: string };
/** A `changes` Read: this UI compared with the window's previous observation, or why there is no comparison. */
@@ -129,11 +127,6 @@ export interface ComputerResult {
process?: ProcessHealth;
/** Complete text of one native element from its observation. */
element_text?: { hint: string; role: string | null; name: string | null; text: string; cut: boolean };
- /** Per-step outcome of a mixed batch; the batch stops at the first step that is not delivered or satisfied. */
- batch?: { total: number; completed: number; stop_reason: string; elapsed_ms: number;
- steps: { index: number; action: string; outcome: string; reason?: string; verdict?: string;
- /** What this step's action changed, when the next walk or page snapshot saw it; the batch's successor covers the last native step. */
- changes?: { changed: number; added: number; removed: number } }[] };
};
isError?: boolean;
}
@@ -752,7 +745,7 @@ export class ComputerSession {
}
const timer = setTimeout(() => { void this.recordStop(); }, seconds * 1000);
this.recording = { timer, started: Date.now() };
- return this.result("delivered", `Recording the main display for up to ${seconds} s. Stop it earlier with computer_record video_stop (or Insert record_stop) to fetch the video.`, state);
+ return this.result("delivered", `Recording the main display for up to ${seconds} s. Stop it earlier with computer_record video_stop to fetch the video.`, state);
});
}
@@ -772,7 +765,7 @@ export class ComputerSession {
recordStop(): Promise<ComputerResult> {
return this.queue(async () => {
const current = this.recording;
- if (!current || !this.host) return this.result("refused", "No recording is running, or it already stopped at its limit; list the session folder with computer_files (or Insert files) to see saved videos.");
+ if (!current || !this.host) return this.result("refused", "No recording is running, or it already stopped at its limit; list the session folder with computer_files to see saved videos.");
clearTimeout(current.timer);
this.recording = undefined;
const native = await this.driver.call("stop_recording", {});
@@ -791,7 +784,7 @@ export class ComputerSession {
try {
const value = await this.host.webview2Debugging(app, enabled);
return this.result("delivered", enabled
- ? `WebView2 debugging is on for ${app} (${value}). It applies when the app next starts, whoever starts it, and stays until turned off; restart the app, then Insert attach_browser into its window.`
+ ? `WebView2 debugging is on for ${app} (${value}). It applies when the app next starts, whoever starts it, and stays until turned off; restart the app, then attach its window with computer_open.`
: `WebView2 debugging is off for ${app}. It applies when the app next starts.`, { operation: "webview2_debugging", app, enabled, value: value ?? null });
} catch (error) { return this.result("refused", `Changing the WebView2 policy failed: ${error instanceof Error ? error.message : error}`); }
});
@@ -1086,7 +1079,7 @@ export class ComputerSession {
try { entry = await this.current(ref, signal); }
catch (error) { return this.result("refused", `Window lifetime check failed: ${error instanceof Error ? error.message : String(error)}`); }
if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window. Discover again.");
- if (!entry.web) return this.result("refused", "DOM is available only in browsers opened with Insert {\"action\":\"open_browser\"} into a machine address, or attached with Insert {\"action\":\"attach_browser\"} into a Chrome or Edge window address (YOLO). Use image/ui/full Reads for other windows.",
+ if (!entry.web) return this.result("refused", "DOM is available only in browsers opened or attached with computer_open (attaching is YOLO). Use image/ui/full Reads for other windows.",
undefined, this.addressed(entry));
return entry;
}
@@ -1286,21 +1279,11 @@ export class ComputerSession {
});
}
- /** Sample browser conditions once without changing the window's observations. Non-browser windows refuse. */
- webCheck(ref: string, conditions: WaitSpec["conditions"], signal?: AbortSignal): Promise<ConditionResult[] | undefined> {
- return this.queue<ConditionResult[] | undefined>(async () => {
- const entry = this.entries.get(ref);
- if (!entry?.web || !this.allowed(entry)) return undefined;
- try { return await entry.web.request<ConditionResult[]>("check", { conditions }, signal); }
- catch { return undefined; }
- });
- }
-
/** The window's latest unannotated capture with pixels, the baseline for comparing its pixels; undefined when none is kept. */
windowFrame(ref: string): WindowFrame | undefined { return this.entries.get(ref)?.frame; }
/**
- * A fresh observation without the human frame, for successors and batch steps: UI-only by default, or with
+ * A fresh observation without the human frame, for successors: UI-only by default, or with
* pixels at a given overview size so they compare with an earlier capture.
*/
observeQuiet(ref: string, signal?: AbortSignal, options: UiCaptureOptions = { granularity: "ui" }): Promise<ComputerResult> {
@@ -1329,7 +1312,8 @@ export class ComputerSession {
try {
const known = this.entries.get(ref);
if (known && !this.allowed(known)) return this.denied(ref);
- if (known) { known.dom = undefined; known.windowState = undefined; known.observation = undefined; known.detail = undefined; }
+ // Sampling state only looks: the window's observations stay current; restore and management retire them.
+ if (known) known.windowState = undefined;
if (signal?.aborted) return this.result("refused", "Cancelled before window state acquisition.");
const entry = await this.current(ref, signal);
if (!entry) return this.result("refused", "Unknown, closed or reused window. Discover again.");
@@ -1347,8 +1331,8 @@ export class ComputerSession {
const id = randomUUID();
entry.windowState = { id, nativeId: state.state_id as string, kind };
return this.result("observed", kind === "management"
- ? "Fresh native control state. One Insert move/resize/minimize/maximize/focus or explicit restore consumes this state. Desktop-frame units are not pointer coordinates."
- : "Fresh native window state. Insert restore explicitly restores and activates this exact window once. No pixels, UI hints, typing or pointer authority.", state,
+ ? "Fresh native window state. computer_window changes it; desktop-frame units are not pointer coordinates."
+ : "Fresh native window state. computer_window restore restores and activates this window.", state,
{ ...this.addressed(entry, kind === "management" ? (state.state as Record<string, unknown>).virtual_desktop : undefined), window_state_id: id, window_state_kind: kind });
} catch (error) { return this.failed(error, false); }
});
@@ -1434,7 +1418,7 @@ export class ComputerSession {
const native = await this.driver.call("parse_visual_regions", { capture_id: observed.captureId, options: { max_regions: 300 } }, signal);
if (native.isError) throw new Error((native.content.filter(c => c.type === "text").map(c => c.text).join(" ") || "parse_visual_regions failed")
+ " The regions view needs the Cua perception extension on this machine; if it is missing, ask the user before installing it "
- + "(about 420 MB) and with their permission Insert {\"action\":\"setup\",\"part\":\"perception\"} into this machine's address.");
+ + "(about 420 MB) and ask them to run /computer-setup perception.");
const parsed = native.structuredContent as { capture: { screenshot: { width: number } }; regions: {
kind: "icon" | "text"; bounds: { x: number; y: number; width: number; height: number }; text?: string | null; label?: string | null; confidence: number }[] };
const scale = (observed.width ?? parsed.capture.screenshot.width) / parsed.capture.screenshot.width;
diff --git a/src/short-addresses.ts b/src/short-addresses.ts
index 1918022..e259f20 100644
--- a/src/short-addresses.ts
+++ b/src/short-addresses.ts
@@ -1,13 +1,13 @@
import { randomInt } from "node:crypto";
/** Address kinds that get a short form, by their one-letter prefix. Displays are a machine's listed displays. */
-const LETTERS = { window: "w", observation: "o", sequence: "s", "window-state": "t", app: "a", sandbox: "b", display: "d" } as const;
+const LETTERS = { window: "w", observation: "o", sequence: "s", app: "a", sandbox: "b", display: "d" } as const;
export type ShortKind = keyof typeof LETTERS;
const KINDS = Object.fromEntries(Object.entries(LETTERS).map(([kind, letter]) => [letter, kind])) as Record<string, ShortKind>;
/** `<letter><number><run tag>`, for example w3qk. */
export const SHORT_REFERENCE = /^([wostabd])([1-9][0-9]{0,6})([a-z]{2})$/;
/** Long addresses this layer shortens in presented text: references, and a machine's listed displays. */
-const LONG = /computer:(window|observation|sequence|window-state|app|sandbox)\/([A-Za-z0-9_-]{1,160})|computer:machine\/([a-z0-9][a-z0-9_-]{0,63})\/display\/(d-[A-Za-z0-9_-]{1,158})/g;
+const LONG = /computer:(window|observation|sequence|app|sandbox)\/([A-Za-z0-9_-]{1,160})|computer:machine\/([a-z0-9][a-z0-9_-]{0,63})\/display\/(d-[A-Za-z0-9_-]{1,158})/g;
/**
* Short, session-local aliases of long computer addresses: `computer:<machine>/<letter><n><tag>`, numbered per
diff --git a/src/standalone-tools.ts b/src/standalone-tools.ts
index a095778..c59bc55 100644
--- a/src/standalone-tools.ts
+++ b/src/standalone-tools.ts
@@ -2,21 +2,17 @@ import { Type, type JsonValue } from "@earendil-works/pi-ai";
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
import { Text } from "@earendil-works/pi-tui";
import type { ComputerResources } from "./computer-resources.ts";
-import { computerActionRenderer } from "./computer-action-renderer.ts";
import { computerReadRenderer } from "./computer-read-renderer.ts";
-import { actionResult, computerNamespace, receiptSchema, registerComputerDelete } from "./computer-tools.ts";
-import {
- COMPUTER_INPUT_HELP, COMPUTER_PATH_HELP, COMPUTER_SEARCH_HELP, COMPUTER_VIEWS_HELP,
- performComputer, presentComputerResult, readComputer,
-} from "./provider.ts";
+import { computerNamespace, registerComputerDelete } from "./computer-tools.ts";
+import { COMPUTER_PATH_HELP, COMPUTER_SEARCH_HELP, COMPUTER_VIEWS_HELP, presentComputerResult, readComputer } from "./provider.ts";
const GUIDE = "docs:computer";
const image = Type.Object({ type: Type.Literal("image"), data: Type.String(), mimeType: Type.String() });
const plain = (value: unknown) => String(value).replace(/[\x00-\x1f\x7f-\x9f]/g, " ");
/**
- * Offer computer use without Pi Agent IDE: the same computer: resources through deferred computer_read, computer_search,
- * computer_write, computer_insert and computer_delete tools that tool_search finds. Tools stay off the model's tool list until a search loads them.
+ * Offer computer use without Pi Agent IDE: computer_read, computer_search and computer_delete stand in for the IDE's Read,
+ * Search and Delete. Like the other computer_* tools they are deferred, off the model's tool list until tool_search loads them.
*/
export function registerStandaloneTools(pi: ExtensionAPI, resources: ComputerResources, guide: string): void {
const namespace = computerNamespace();
@@ -69,29 +65,5 @@ export function registerStandaloneTools(pi: ExtensionAPI, resources: ComputerRes
return { content: [{ type: "text", text: resources.shorten(presentComputerResult(result)) }], details: result.details, ...(result.isError ? { isError: true } : {}) };
},
});
- for (const tool of ["write", "insert"] as const) {
- const field = tool === "write" ? "content" : "text";
- const renderer = computerActionRenderer(tool);
- pi.registerTool({
- name: `computer_${tool}`, label: `Computer ${tool}`, exposure: "deferred", namespace,
- description: tool === "write"
- ? "Use computer_write to type literal text into the app window of a fresh observation address."
- : "Use computer_insert to send one JSON command: an input action into an observation, open/close/browser/sandbox commands into a machine, or wait/verify into a window.",
- parameters: Type.Object({
- path: Type.String({ description: tool === "write"
- ? "A fresh computer:observation/... address, optionally with ?hint=...&delivery=foreground&delay_ms=0."
- : "The computer: address that receives the command: an observation, window, window-state, display, machine, sandbox or computer:session." }),
- [field]: Type.String({ description: tool === "write" ? "Literal text to type. Nothing is selected, submitted or executed." : COMPUTER_INPUT_HELP }),
- }, { additionalProperties: false }),
- outputSchema: receiptSchema,
- annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },
- renderCall: args => new Text(`computer_${tool} ${plain((args as { path: string }).path)}`, 0, 0),
- renderResult: renderer.renderResult,
- async execute(_id, params, signal, _update, ctx) {
- const input = params as Record<string, unknown>;
- return actionResult(await performComputer(resources, tool, String(input.path), input, ctx.cwd, signal));
- },
- });
- }
registerComputerDelete(pi, resources);
}
diff --git a/src/web-browser.ts b/src/web-browser.ts
index 738c478..91c345d 100644
--- a/src/web-browser.ts
+++ b/src/web-browser.ts
@@ -283,7 +283,7 @@ const tabId = (value: unknown): string => {
if (typeof value !== "string" || !/^t\d{1,6}$/.test(value)) throw Error("tab must be a tab id such as t2 from the tab list.");
return value;
};
-/** Validate one Insert command on a DOM observation before anything reaches the browser. */
+/** Validate one page action before anything reaches the browser. */
export function validateWebAction(value: unknown): WebAction {
const command = record(value, "DOM action");
const name = command.action as WebAction["action"];
diff --git a/src/web-worker.ts b/src/web-worker.ts
index f366e64..5dcc6cd 100644
--- a/src/web-worker.ts
+++ b/src/web-worker.ts
@@ -112,7 +112,7 @@ function listTabs(): WebTab[] {
...(tab.opener ? { opener: tab.opener } : {}), ...(tab.dialog ? { dialog: tab.dialog.type() } : {}) }));
}
function current(): Tab {
- if (!active) throw new Error("No tab is active. Insert {\"action\":\"switch_tab\",\"tab\":\"t1\"} with an id from the tab list; nothing was sent.");
+ if (!active) throw new Error("No tab is active. Choose one with computer_browser switch_tab and an id from the tab list; nothing was sent.");
return active;
}
function idle(tab: Tab): Page {
@@ -186,8 +186,8 @@ async function open({ browser, url, profile: kind, attach: target }: WebLaunchOp
context = browser === "firefox"
? await firefox.launchPersistentContext(directory, { headless: false, viewport: null }).catch((error: Error) => {
if (!/Executable doesn't exist/.test(error.message)) throw error;
- throw new Error("Playwright's Firefox is not installed on this machine. Ask the user before installing it; with their permission "
- + "Insert {\"action\":\"setup\",\"part\":\"firefox\"} into this machine's address, then open the browser again.");
+ throw new Error("Playwright's Firefox is not installed on this machine. Ask the user to install it "
+ + "with /computer-setup firefox, then open the browser again.");
})
: await chromium.launchPersistentContext(directory, {
channel: browser, headless: false, chromiumSandbox: true, viewport: null, ignoreDefaultArgs: true,
@@ -304,7 +304,7 @@ async function attach({ pid, title }: { pid: number; title: string }) {
catch (error) { throw new Error(`VS Code's browser bridge did not accept the connection: ${error instanceof Error ? error.message.split("\n")[0] : error}. Open an integrated browser tab (Browser: Open Integrated Browser) and attach again. Nothing was attached.`); }
via = "vscode";
} else if (!found) {
- throw new Error(`No Chromium debugging endpoint in this window's processes (${[...new Set(tree.processes.map(item => item.name))].slice(0, 6).join(", ")}). Start an Electron or CEF app with --remote-debugging-port=0 in open's args; for a WebView2 app, Insert {"action":"webview2_debugging","app":"<exe name>","enabled":true} into its machine and start the app again; for VS Code, install the browser bridge (docs:computer). Nothing was attached.`);
+ throw new Error(`No Chromium debugging endpoint in this window's processes (${[...new Set(tree.processes.map(item => item.name))].slice(0, 6).join(", ")}). Start an Electron or CEF app with --remote-debugging-port=0 in open's args; for a WebView2 app, set computer_open webview2_debugging on its machine and start the app again; for VS Code, install the browser bridge (docs:computer). Nothing was attached.`);
} else if (togglePort && togglePath?.startsWith("/devtools/browser/")) {
// Chrome asks "Allow remote debugging?" before it completes the handshake, so the connect waits for approval.
try { attached = await chromium.connectOverCDP(`ws://127.0.0.1:${togglePort}${togglePath}`, { timeout: ATTACH_APPROVAL_MS }); }
diff --git a/src/windows-node.ts b/src/windows-node.ts
index d21452d..7abb0d0 100644
--- a/src/windows-node.ts
+++ b/src/windows-node.ts
@@ -19,6 +19,6 @@ const exists = (file: string) => access(file).then(() => true, () => false);
export async function findWindowsNode(interop?: string, deps = { localAppData: windowsLocalAppData, exists }): Promise<string> {
const candidates = [portableNodePath(await deps.localAppData(interop)), INSTALLED_NODE];
for (const candidate of candidates) if (await deps.exists(candidate)) return candidate;
- throw new Error("Browsers need Node.js on Windows, and none is installed here. Ask the user before installing it; "
- + "with their permission Insert {\"action\":\"setup\",\"part\":\"browser\"} into this machine's address (about 30 MB into their own folders), then open the browser again.");
+ throw new Error("Browsers need Node.js on Windows, and none is installed here. Ask the user to install it "
+ + "with /computer-setup browser (about 30 MB into their own folders), then open the browser again.");
}
diff --git a/test/action-changes.test.ts b/test/action-changes.test.ts
index 015359e..adb0e24 100644
--- a/test/action-changes.test.ts
+++ b/test/action-changes.test.ts
@@ -51,16 +51,18 @@ async function fixture() {
const read = async (views = ["ui"]) => {
const view = (await resources.read(address(window), cwd, views))!;
const observation = computerSource("observation", view.details.observation_id!);
- return { view, observation: address(observation), at: (index: number) => address(`${observation}?hint=${encodeURIComponent(view.details.ui!.elements[index].hint)}`) };
+ return { view, observation: address(observation), at: (index: number) => ({ ...address(observation), hint: view.details.ui!.elements[index].hint }) };
};
const walks = () => vi.mocked(driver.call).mock.calls.filter(([name]) => name === "get_window_state").length;
- return { resources, window, read, walks };
+ // The app paints by itself, as an animation or another person would.
+ const paint = (x: number, y: number) => { dots.push({ x, y }); };
+ return { resources, window, read, walks, paint };
}
const images = (result: ComputerResult) => result.content.filter((block): block is Extract<Content, { type: "image" }> => block.type === "image");
test("an action on an observation with pixels reports the changed area with before and after crops", async () => {
const f = await fixture(), { at } = await f.read(["full"]);
- const result = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }), "insert");
+ const result = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }));
expect(result.details.successor?.diff?.pixels).toMatchObject({ box: { x: 20, y: 40, width: 10, height: 10 }, regions: 1, crops: [{ x: 8, y: 28, width: 34, height: 34 }] });
expect(presentComputerResult(result)).toMatch(/~ \d+:a Button "Count 0" → "Count 1"\nPixels changed in 1 area \(overview pixels; 0\.06% of the window\):\n x=20 y=40 w=10 h=10; images 1 and 2 show x=8 y=28 w=34 h=34 before and after\n/);
const [before, after] = images(result);
@@ -70,38 +72,39 @@ test("an action on an observation with pixels reports the changed area with befo
test("pixel input on a canvas reports only its pixel change, and empty space reports nothing", async () => {
const f = await fixture(), { observation } = await f.read(["full"]);
- const stroke = await f.resources.perform(observation, cwd, JSON.stringify({ action: "click", x: 300, y: 200 }), "insert");
+ const stroke = await f.resources.perform(observation, cwd, JSON.stringify({ action: "click", x: 300, y: 200 }));
expect(stroke.details.successor?.diff).toMatchObject({ counts: { changed: 0, added: 0, removed: 0 }, pixels: { box: { x: 297, y: 197, width: 6, height: 6 } } });
expect(images(stroke)).toHaveLength(2);
const next = address(computerSource("observation", stroke.details.successor!.observation_id!));
- const empty = await f.resources.perform(next, cwd, JSON.stringify({ action: "click", x: 300, y: 60 }), "insert");
+ const empty = await f.resources.perform(next, cwd, JSON.stringify({ action: "click", x: 300, y: 60 }));
expect(presentComputerResult(empty)).toMatch(/Nothing changed in the UI or the window's pixels within \d+ ms after the action\./);
expect(images(empty)).toHaveLength(0);
});
test("an action on a UI-only observation says that pixels were not compared, and its successor has them", async () => {
const f = await fixture(), { at } = await f.read();
- const first = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }), "insert");
+ const first = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }));
expect(presentComputerResult(first)).toMatch(/Pixels were not compared: the input observation had no pixels\./);
const view = (await f.resources.read(address(computerSource("observation", first.details.successor!.observation_id!)), cwd, []))!;
const hint = view.details.ui!.elements[0].hint;
- const second = await f.resources.perform(address(`${computerSource("observation", first.details.successor!.observation_id!)}?hint=${hint}`), cwd, JSON.stringify({ action: "click" }), "insert");
+ const second = await f.resources.perform({ ...address(computerSource("observation", first.details.successor!.observation_id!)), hint } as ReturnType<typeof address>, cwd, JSON.stringify({ action: "click" }));
expect(second.details.successor?.diff?.pixels?.box).toEqual({ x: 32, y: 40, width: 10, height: 10 });
});
test("a changes Read compares the window's pixels with its previous observation", async () => {
- const f = await fixture(), { observation } = await f.read(["full"]);
- await f.resources.perform(observation, cwd, JSON.stringify({ action: "click", x: 300, y: 200, successor: false }), "insert");
+ const f = await fixture();
+ await f.read(["full"]);
+ f.paint(200, 150);
const changes = (await f.resources.read(address(f.window), cwd, ["changes"]))!;
- expect(changes.details.changes?.diff?.pixels?.box).toEqual({ x: 297, y: 197, width: 6, height: 6 });
- expect(presentComputerResult(changes)).toMatch(/Pixels changed in 1 area .*\n x=297 y=197 w=6 h=6;/);
+ expect(changes.details.changes?.diff?.pixels?.box).toEqual({ x: 197, y: 147, width: 6, height: 6 });
+ expect(presentComputerResult(changes)).toMatch(/Pixels changed in 1 area .*\n x=197 y=147 w=6 h=6;/);
// Only the two crops, not the full overview.
expect(images(changes)).toHaveLength(2);
});
test("an action reports what it changed by default, and its successor is the current observation", async () => {
const f = await fixture(), { at } = await f.read();
- const result = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }), "insert");
+ const result = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }));
const shown = presentComputerResult(result);
expect(shown).toMatch(/Changes: 1 changed, 0 added, 0 removed\n~ \d+:a Button "Count 0" → "Count 1"/);
expect(f.resources.grantsAuthority(address(computerSource("observation", result.details.successor!.observation_id!)))).toBe(true);
@@ -109,34 +112,14 @@ test("an action reports what it changed by default, and its successor is the cur
test("an action that changes nothing says so after looking again for a moment", async () => {
const f = await fixture(), { at } = await f.read();
- const result = await f.resources.perform(at(1), cwd, JSON.stringify({ action: "click" }), "insert");
+ const result = await f.resources.perform(at(1), cwd, JSON.stringify({ action: "click" }));
expect(result.details.successor?.diff?.waited_ms).toBeGreaterThanOrEqual(300);
expect(presentComputerResult(result)).toMatch(/Nothing changed in the UI within \d+ ms after the action\. Check that the target was right/);
});
-test("successor false turns the change list off for one call", async () => {
- const f = await fixture(), { at } = await f.read();
- const before = f.walks();
- const result = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click", successor: false }), "insert");
- expect(result.details.status).toBe("delivered");
- expect(result.details.successor).toBeUndefined();
- expect(f.walks()).toBe(before);
-});
-
test("a locator action on the window address compares with the walk that resolved its target", async () => {
const f = await fixture();
- const result = await f.resources.perform(address(f.window), cwd, JSON.stringify({ action: "click", target: "getByRole('button', { name: /Count/ })" }), "insert");
+ const result = await f.resources.perform(address(f.window), cwd, JSON.stringify({ action: "click", target: "getByRole('button', { name: /Count/ })" }));
expect(result.details.successor?.diff?.changed).toEqual([expect.stringMatching(/Button "Count 0" → "Count 1"$/)]);
});
-test("each batch step says what it changed, and the batch reports the change from its start", async () => {
- const f = await fixture(), { view } = await f.read();
- const result = await f.resources.perform(address(computerSource("observation", view.details.observation_id!)), cwd, JSON.stringify({ action: "batch", steps: [
- { action: "click", target: "getByRole('button', { name: /Count/ })" },
- { action: "click", target: "getByText('Status')" },
- { action: "click", target: "getByRole('button', { name: /Count/ })" },
- ] }), "insert");
- expect(result.details.batch?.steps.map(step => step.changes)).toEqual([{ changed: 1, added: 0, removed: 0 }, { changed: 0, added: 0, removed: 0 }, undefined]);
- expect(presentComputerResult(result)).toMatch(/Step 0: click → delivered; 1 changed, 0 added, 0 removed\nStep 1: click → delivered; nothing changed\n/);
- expect(result.details.successor?.diff?.changed).toEqual([expect.stringMatching(/Button "Count 0" → "Count 2"$/)]);
-});
diff --git a/test/computer-address.test.ts b/test/computer-address.test.ts
index 24f3d53..2b753cf 100644
--- a/test/computer-address.test.ts
+++ b/test/computer-address.test.ts
@@ -12,10 +12,9 @@ describe("computer resource addresses", () => {
expect(parseComputerAddress("computer:machine/local/windows")).toEqual({ kind: "machine", ref: "local", part: "windows", source: "computer:machine/local/windows" });
expect(parseComputerAddress("computer:machine/lab")).toEqual({ kind: "machine", ref: "lab", source: "computer:machine/lab" });
expect(parseComputerAddress("computer:window/window-1")).toEqual({ kind: "window", ref: "window-1", source: "computer:window/window-1" });
- expect(parseComputerAddress("computer:observation/snapshot-1?hint=1:a&delivery=foreground&delay_ms=0")).toEqual({
- kind: "observation", ref: "snapshot-1", source: "computer:observation/snapshot-1?hint=1:a&delivery=foreground&delay_ms=0",
- hint: "1:a", delivery: "foreground", delayMs: 0,
- });
+ expect(parseComputerAddress("computer:observation/snapshot-1")).toEqual({ kind: "observation", ref: "snapshot-1", source: "computer:observation/snapshot-1" });
+ // Input options went with Write and Insert: the input tools take refs and delivery as fields.
+ for (const query of ["?hint=1:a", "?delivery=foreground", "?delay_ms=0"]) expect(() => parseComputerAddress(`computer:observation/snapshot-1${query}`)).toThrow();
});
it("rejects identity overrides, ambiguous parameters and broadening paths", () => {
diff --git a/test/computer-presentation.test.ts b/test/computer-presentation.test.ts
index 1f02d53..b1dc53e 100644
--- a/test/computer-presentation.test.ts
+++ b/test/computer-presentation.test.ts
@@ -11,7 +11,7 @@ test("control-state presentation separates desktop frame coordinates from pointe
foreground_at_target: false, keyboard_focus_at_target: null, virtual_desktop: { on_current_desktop: null },
} } } };
const before = JSON.stringify(result), text = presentComputerResult(result);
- expect(text).toContain("computer:window-state/management-state");
+ expect(text).not.toContain("computer:window-state/");
expect(text).toContain('{"x":-100,"y":80,"width":640,"height":800}');
expect(text).toContain("source=dwm_visible_frame; unit=windows_desktop_pixels");
expect(text).toContain("maximized=true");
@@ -59,14 +59,14 @@ test("schema-rejected management retains an explicitly unvalidated prefix withou
expect(text).not.toContain("Computer delivered");
expect(JSON.stringify(result)).toBe(before);
});
-test("window state exposes its restore-only address and independent foreground/focus metadata", () => {
+test("window state shows independent foreground/focus metadata and no state address", () => {
const result: ComputerResult = { content: [], details: { status: "observed", mode: "yolo", window_state_id: "fresh-state",
state: { minimized: true, visible: true, foreground_at_target: false, keyboard_focus_at_target: null } } };
const text = presentComputerResult(result);
- expect(text).toContain("computer:window-state/fresh-state");
+ expect(text).not.toContain("computer:window-state/");
expect(text).toContain("minimized=true, visible=true");
expect(text).toContain("foreground=false, keyboard_focus=null");
- expect(text).toContain("restore");
+ expect(text).toContain("computer_window");
expect(text).not.toContain("computer:observation/");
});
diff --git a/test/computer-resources.test.ts b/test/computer-resources.test.ts
index 7c82651..ae9c567 100644
--- a/test/computer-resources.test.ts
+++ b/test/computer-resources.test.ts
@@ -2,6 +2,7 @@ import { expect, test, vi } from "vitest";
import sharp from "sharp";
import { computerSource } from "../src/computer-address.ts";
import { ComputerResources, requireComputerAddress } from "../src/computer-resources.ts";
+import { typePlan } from "../src/computer-tools.ts";
import { ComputerSession, type ComputerMode, type Content, type Driver, type DriverResult } from "../src/session.ts";
const cwd = "/synthetic-computer-fixture";
@@ -56,7 +57,7 @@ async function fixture() {
setBounds(value: typeof liveBounds) { liveBounds = value; } };
}
-test("literal Write preserves all text and descriptor/UI reads do not refresh native grounding", async () => {
+test("typed text is sent exactly and descriptor/UI reads do not refresh native grounding", async () => {
const f = await fixture(), view = await f.read(["ui"]);
const before = f.native().length;
expect(f.resources.descriptor(view.observation)).toContain(view.source);
@@ -64,8 +65,8 @@ test("literal Write preserves all text and descriptor/UI reads do not refresh na
expect((await f.resources.search(view.observation, cwd, "Text field"))?.details.status).toBe("observed");
expect(f.native()).toHaveLength(before);
const literal = ' {"looks":"like JSON"}\n"quoted text"\n ';
- const target = address(`${view.source}?hint=${encodeURIComponent(view.hint!)}&delivery=foreground&delay_ms=0`);
- const written = await f.resources.perform(target, cwd, literal, "write");
+ const target = { ...view.observation, hint: view.hint! } as typeof view.observation;
+ const written = await f.resources.perform(target, cwd, JSON.stringify({ action: "type_text", text: literal, delivery_mode: "foreground", delay_ms: 0 }));
expect(written.details.status).toBe("delivered");
// The only captures after input are the successor's UI walks; the unchanged fixture UI reads as no change.
expect(written.details.successor?.diff?.counts).toEqual({ changed: 0, added: 0, removed: 0 });
@@ -75,7 +76,7 @@ test("literal Write preserves all text and descriptor/UI reads do not refresh na
target: { kind: "window", pid: 42, window_id: 91 }, delivery_mode: "foreground",
element_token: "snapshot-1:0", text: literal, delay_ms: 0,
}, undefined]]);
- await expect(f.resources.perform(target, cwd, "replay", "write")).rejects.toThrow(/expired/i);
+ await expect(f.resources.perform(target, cwd, JSON.stringify({ action: "type_text", text: "replay" }))).rejects.toThrow(/expired/i);
expect(f.inputs()).toHaveLength(1); expect(f.captures()).toHaveLength(captured);
});
@@ -83,13 +84,13 @@ test.each(["refused", "unknown"] as const)("a native %s receipt consumes the pro
const f = await fixture(), view = await f.read(["ui"]);
f.setOutcome({ isError: true, content: [{ type: "text", text: "Native outcome" }], structuredContent: { status } });
const command = JSON.stringify({ action: "click", hint: view.hint });
- const result = await f.resources.perform(view.observation, cwd, command, "insert");
+ const result = await f.resources.perform(view.observation, cwd, command);
expect(result.details.status).toBe(status);
// A refusal sent nothing, so nothing is re-read; an unknown outcome gets a successor to show what may have happened.
if (status === "refused") expect(f.captures()).toHaveLength(1);
else expect(result.details.successor?.observation_id).toBeDefined();
const captured = f.captures().length;
- await expect(f.resources.perform(view.observation, cwd, command, "insert")).rejects.toThrow(/expired/i);
+ await expect(f.resources.perform(view.observation, cwd, command)).rejects.toThrow(/expired/i);
expect(f.inputs()).toHaveLength(1); expect(f.captures()).toHaveLength(captured);
});
@@ -115,15 +116,16 @@ test("per-Read native cap is forwarded without changing the session defaults", a
test("temporal Read retires input authority and frame selection does not recapture", async () => {
const f = await fixture(), view = await f.read();
- const captured = await f.resources.read(f.window, cwd, ['sequence:{"duration_ms":1,"max_frames":1}']);
+ await expect(f.resources.read(f.window, cwd, ['sequence:{"duration_ms":1,"max_frames":1}'])).rejects.toThrow(/computer_record/);
+ const captured = await f.resources.sequence(f.window, cwd, ['sequence:{"duration_ms":1,"max_frames":1}']);
expect(captured?.details.sequence?.frames).toHaveLength(1);
- await expect(f.resources.perform(view.observation, cwd, "stale", "write")).rejects.toThrow(/expired/i);
+ await expect(f.resources.perform(view.observation, cwd, JSON.stringify({ action: "type_text", text: "stale" }))).rejects.toThrow(/expired/i);
const sequence = address(computerSource("sequence", captured!.details.sequence!.sequence_id));
const before = f.captures().length;
const frames = await f.resources.read(sequence, cwd, ["frames:first"]);
expect(frames?.content.some(block => block.type === "image")).toBe(true);
expect(frames?.details.selected_frames?.[0].index).toBe(0);
- await expect(f.resources.perform(sequence, cwd, "not authority", "write")).rejects.toThrow(/observation address/i);
+ await expect(f.resources.perform(sequence, cwd, JSON.stringify({ action: "type_text", text: "not authority" }))).rejects.toThrow(/observation address/i);
expect(f.captures()).toHaveLength(before);
expect(f.inputs()).toHaveLength(0);
});
@@ -132,9 +134,9 @@ test("a new overview retires the prior resource instead of rebinding its hint",
const f = await fixture(), first = await f.read(["ui"]), latest = await f.read(["ui"]);
expect(latest.source).not.toBe(first.source);
await expect(f.resources.read(first.observation, cwd, ["ui"])).rejects.toThrow(/expired/i);
- await expect(f.resources.perform(first.observation, cwd, "stale", "write")).rejects.toThrow(/expired/i);
+ await expect(f.resources.perform(first.observation, cwd, JSON.stringify({ action: "type_text", text: "stale" }))).rejects.toThrow(/expired/i);
expect(f.inputs()).toHaveLength(0);
- expect((await f.resources.perform(address(`${latest.source}?hint=${encodeURIComponent(latest.hint!)}`), cwd, "current", "write")).details.status).toBe("delivered");
+ expect((await f.resources.perform({ ...latest.observation, hint: latest.hint! } as typeof latest.observation, cwd, JSON.stringify({ action: "type_text", text: "current" }))).details.status).toBe("delivered");
expect(f.inputs()[0][1]).toMatchObject({ element_token: "snapshot-2:0", text: "current" });
// Two overviews, then the input, then the successor's captures.
const order = f.native().map(([name]) => name).filter(name => name !== "list_windows" && name !== "get_window_control_state");
@@ -149,7 +151,7 @@ test.each(["safe", "closed", "reused"] as const)("%s targets refuse cached reads
const checked = await f.resources.read(view.observation, cwd, ["ui"]);
expect(checked?.details.status).toBe("refused");
expect(checked?.content.every(block => block.type === "text")).toBe(true);
- await expect(f.resources.perform(view.observation, cwd, "forbidden", "write")).rejects.toThrow(/expired/i);
+ await expect(f.resources.perform(view.observation, cwd, JSON.stringify({ action: "type_text", text: "forbidden" }))).rejects.toThrow(/expired/i);
expect(f.inputs()).toHaveLength(0); expect(f.captures()).toHaveLength(1);
});
@@ -157,18 +159,18 @@ test.each(["safe", "closed", "reused"] as const)("%s after reading refuses direc
const f = await fixture(), view = await f.read(["ui"]);
if (change === "safe") f.setMode("safe");
else f.setLifetime(change === "closed" ? undefined : "different-native-life");
- expect((await f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", hint: view.hint }), "insert")).details.status).toBe("refused");
+ expect((await f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", hint: view.hint }))).details.status).toBe("refused");
expect(f.inputs()).toHaveLength(0); expect(f.captures()).toHaveLength(1);
});
-test("display resources and their observations never grant Write or Insert authority", async () => {
+test("display resources and their observations never take input", async () => {
const f = await fixture(), display = address("computer:machine/local/display/primary");
const result = await f.resources.read(display, cwd);
expect(result?.details.status).toBe("observed");
const observed = address(computerSource("observation", result!.details.agent_image!.image_id as string));
for (const target of [display, observed]) {
- await expect(f.resources.perform(target, cwd, "not input", "write")).rejects.toThrow(/observation|readonly/i);
- await expect(f.resources.perform(target, cwd, JSON.stringify({ action: "click", x: 10, y: 10 }), "insert")).rejects.toThrow(/observation|readonly|only resize/i);
+ await expect(f.resources.perform(target, cwd, JSON.stringify({ action: "type_text", text: "not input" }))).rejects.toThrow(/observation|readonly|only resize/i);
+ await expect(f.resources.perform(target, cwd, JSON.stringify({ action: "click", x: 10, y: 10 }))).rejects.toThrow(/observation|readonly|only resize/i);
}
expect(f.inputs()).toHaveLength(0); expect(f.captures()).toHaveLength(1);
f.setMode("safe");
@@ -186,7 +188,7 @@ test("a successful rectangle Read is readonly detail and consumes its overview w
expect(detail?.details).not.toHaveProperty("observation_id");
expect(f.captures()).toHaveLength(1);
expect(f.native().filter(([name]) => name === "zoom")).toEqual([["zoom", { pid: 42, window_id: 91, x1: 80, y1: 60, x2: 120, y2: 80 }, undefined]]);
- await expect(f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", x: 100, y: 70 }), "insert")).rejects.toThrow(/expired/i);
+ await expect(f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", x: 100, y: 70 }))).rejects.toThrow(/expired/i);
expect(f.inputs()).toHaveLength(0);
});
@@ -194,19 +196,19 @@ test("invalid JSON and identity overrides are rejected before native input and c
const f = await fixture(), view = await f.read(["ui"]);
for (const value of ["not JSON", "[]", JSON.stringify({ action: "click", hint: view.hint, window_ref: "other-window" }),
JSON.stringify({ action: "click", hint: view.hint, observation_id: "other-observation" })])
- await expect(f.resources.perform(view.observation, cwd, value, "insert")).rejects.toThrow();
- const hinted = address(`${view.source}?hint=${encodeURIComponent(view.hint!)}`);
- await expect(f.resources.perform(hinted, cwd, JSON.stringify({ action: "click", x: 100, y: 70 }), "insert")).rejects.toThrow(/target|hint/i);
+ await expect(f.resources.perform(view.observation, cwd, value)).rejects.toThrow();
+ const hinted = { ...view.observation, hint: view.hint! } as typeof view.observation;
+ await expect(f.resources.perform(hinted, cwd, JSON.stringify({ action: "click", x: 100, y: 70 }))).rejects.toThrow(/target|hint/i);
expect(f.inputs()).toHaveLength(0); expect(f.captures()).toHaveLength(1);
- expect((await f.resources.perform(hinted, cwd, "valid literal", "write")).details.status).toBe("delivered");
+ expect((await f.resources.perform(hinted, cwd, JSON.stringify({ action: "type_text", text: "valid literal" }))).details.status).toBe("delivered");
expect(f.inputs()[0][1]).toMatchObject({ target: { kind: "window", pid: 42, window_id: 91 }, text: "valid literal" });
});
test("pre-dispatch cancellation cannot send input or leave a reusable provider handle", async () => {
const f = await fixture(), view = await f.read(["ui"]), cancellation = new AbortController();
cancellation.abort();
- expect((await f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", hint: view.hint }), "insert", cancellation.signal)).details.status).toBe("refused");
- await expect(f.resources.perform(view.observation, cwd, "retry", "write")).rejects.toThrow(/expired/i);
+ expect((await f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", hint: view.hint }), cancellation.signal)).details.status).toBe("refused");
+ await expect(f.resources.perform(view.observation, cwd, JSON.stringify({ action: "type_text", text: "retry" }))).rejects.toThrow(/expired/i);
expect(f.inputs()).toHaveLength(0); expect(f.captures()).toHaveLength(1);
});
@@ -218,22 +220,21 @@ test.each(["native-partial", "in-flight-cancel"] as const)("%s preserves the att
{ action: "type_text", hint: view.hint, text: "first prefix", delay_ms: 0 },
{ action: "key", key: "return" },
] });
- const result = await f.resources.perform(view.observation, cwd, command, "insert", cancellation.signal);
+ const result = await f.resources.perform(view.observation, cwd, command, cancellation.signal);
// A cancelled sequence lets its current step finish and reports it; a native partial step stays unknown.
expect(result.details.status).toBe(interruption === "native-partial" ? "unknown" : "refused");
expect(result.details.state).toMatchObject({ sequence: { total: 2, attempted: 1, completed: interruption === "native-partial" ? 0 : 1, steps: [{ action: "type_text" }] } });
expect(f.inputs()).toHaveLength(1); expect(f.inputs()[0][1]).toMatchObject({ text: "first prefix" });
const captured = f.captures().length;
- await expect(f.resources.perform(view.observation, cwd, command, "insert")).rejects.toThrow(/expired/i);
+ await expect(f.resources.perform(view.observation, cwd, command)).rejects.toThrow(/expired/i);
expect(f.inputs()).toHaveLength(1); expect(f.captures()).toHaveLength(captured);
});
-test("an address delivery preference does not block semantic actions", async () => {
+test("foreground delivery does not block semantic actions", async () => {
const f = await fixture();
(f.driver as Driver & { semanticActions: boolean }).semanticActions = true;
const view = await f.read();
- const hinted = address(`${view.source}?hint=${encodeURIComponent(view.hint!)}&delivery=foreground`);
- await f.resources.perform(hinted, cwd, JSON.stringify({ action: "set_value", text: "13" }), "insert");
+ await f.resources.input(f.window, cwd, typePlan({ window: f.window.source, target: view.hint!, text: "13", replace: true, foreground: true }));
expect(f.inputs().map(([name]) => name)).toEqual(["semantic_action"]);
expect(f.inputs()[0][1]).toMatchObject({ operation: "set_value", value: "13" });
expect(f.inputs()[0][1]).not.toHaveProperty("delivery_mode");
@@ -248,26 +249,3 @@ test("a display overview accepts a size for that Read only", async () => {
await expect(f.resources.read(display, cwd, ['capture:{"max_elements":5}'])).rejects.toThrow(/accepts only capture/);
});
-test("pixel-only input without a successor keeps the observation until the window size changes; UI hints and other input spend it", async () => {
- const f = await fixture();
- (f.driver as Driver & { windowManagement: boolean }).windowManagement = true;
- const view = await f.read();
- // A default successor would replace the observation; without one, pixel-only input keeps it for more pixel input.
- const click = (x: number) => f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", x, y: 40, successor: false }), "insert");
- for (const x of [10, 20, 30]) expect((await click(x)).details.status).toBe("delivered");
- expect(f.captures()).toHaveLength(1);
- // The native capture is single-use; kept observations send window-local pixels without it.
- expect(f.inputs()[0][1]).toMatchObject({ capture_id: "capture-1" });
- expect(f.inputs()[1][1]).not.toHaveProperty("capture_id");
- const hinted = await f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", hint: view.hint }), "insert");
- expect(hinted.details.status).toBe("refused");
- expect(hinted.content[0]).toMatchObject({ text: expect.stringContaining("UI hints of this observation expired") });
- const again = await f.read();
- f.setBounds({ x: 300, y: 90, width: 500, height: 320 });
- expect((await f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5, successor: false }), "insert")).details.status).toBe("delivered");
- f.setBounds({ x: 300, y: 90, width: 640, height: 320 });
- const resized = await f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5, successor: false }), "insert");
- expect(resized.details.status).toBe("refused");
- expect(resized.content[0]).toMatchObject({ text: expect.stringContaining("resized from 500x320 to 640x320") });
- await expect(f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5 }), "insert")).rejects.toThrow(/expired/i);
-});
diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.ts
index 370a4c8..d6f0093 100644
--- a/test/computer-tools.test.ts
+++ b/test/computer-tools.test.ts
@@ -1,5 +1,6 @@
import { expect, test, vi } from "vitest";
import { requireComputerAddress, ComputerResources, refKind } from "../src/computer-resources.ts";
+import { computerEditRefusal } from "../src/provider.ts";
import { clickPlan, clipboardCommand, dragPlan, filesCommand, hoverPlan, openCommand, pressPlan, recordCommand, scrollPlan, selectPlan, timelinePlan, typePlan, waitCommand, windowCommand } from "../src/computer-tools.ts";
import type { WebBrowser, WebLauncher } from "../src/web-browser.ts";
import type { MachineHost } from "../src/machine-host.ts";
@@ -272,7 +273,7 @@ test("computer_record video_stop says where it fetched the video in the summary
const resources = new ComputerResources(async () => session, async () => session.close());
const workspace = await mkdtemp(join(tmpdir(), "pi-record-"));
const run = (command: Record<string, unknown>) => computerAction(resources, "computer:machine/local", String(command.action), () => at("computer:machine/local"),
- address => resources.perform(address, workspace, JSON.stringify(command), "insert"));
+ address => resources.perform(address, workspace, JSON.stringify(command)));
try {
expect((await run({ action: "record_start", seconds: 30 })).summary).toContain("Stop it earlier with computer_record video_stop");
const stopped = await run({ action: "record_stop", to: "out/run.mp4" });
@@ -284,4 +285,14 @@ test("computer_record video_stop says where it fetched the video in the summary
await session.close();
await rm(workspace, { recursive: true, force: true });
}
-});
\ No newline at end of file
+});
+test("Write and Insert on a computer address refuse and name the tool that does the operation now", () => {
+ expect(computerEditRefusal("write", { path: "computer:observation/o1", content: "hi" })).toMatch(/^Write does not act on computer resources; nothing was sent\. Use computer_type\./);
+ for (const [action, tool] of [["click", "computer_click"], ["fill", "computer_type"], ["hotkey", "computer_press"], ["menu", "computer_select"],
+ ["open_browser", "computer_open"], ["restore", "computer_window"], ["verify", "computer_wait"], ["navigate", "computer_browser"],
+ ["put", "computer_files"], ["clipboard_read", "computer_clipboard"], ["record_start", "computer_record"], ["kill", "Delete"],
+ ["setup", "/computer-setup"], ["batch", "Codemode"]])
+ expect(computerEditRefusal("insert", { path: "computer:machine/local", text: JSON.stringify({ action }) })).toContain(`Use ${tool}`);
+ // Unknown or unreadable commands still learn where the tools are.
+ expect(computerEditRefusal("insert", { path: "computer:session", text: "not json" })).toMatch(/^Insert does not act on computer resources; nothing was sent\. Each operation has its own computer_\* tool: tool_search/);
+});
diff --git a/test/isolated-session.test.ts b/test/isolated-session.test.ts
index 51cfdeb..747c2e1 100644
--- a/test/isolated-session.test.ts
+++ b/test/isolated-session.test.ts
@@ -61,7 +61,7 @@ test("a session provisions, starts its driver through the reverse relay, and clo
test("a machine without MXC refuses before provisioning anything", async () => {
const f = await fakeHost({ installed: false });
- await expect(IsolatedSession.open(f.host, "/mnt/c/runtime/cua-driver.exe")).rejects.toThrow(/MXC is not installed.*Ask the user.*"part":"mxc"/);
+ await expect(IsolatedSession.open(f.host, "/mnt/c/runtime/cua-driver.exe")).rejects.toThrow(/MXC is not installed.*Ask the user.*\/computer-setup mxc/);
expect(f.phases).toEqual([]);
});
diff --git a/test/live/remote.ts b/test/live/remote.ts
index 573d951..beabd96 100644
--- a/test/live/remote.ts
+++ b/test/live/remote.ts
@@ -31,15 +31,29 @@ export async function connectLive(cwd = process.cwd()) {
// A machine starts on its first use; the live checks use the configured one unless LIVE_MACHINE=local.
const machine = computerSource("machine", name);
const runner = name === "local" ? localRunner(config.interop) : sshRunner((await machineConfig(cwd, name)).ssh);
+ /**
+ * One engine command, as the computer_* tools send it: `?hint=` and `?delivery=` on an observation become its ref and
+ * delivery_mode, close goes to Delete and kill to Delete with ?force=true, as the tools do.
+ */
+ const engine = (path: string, command: Record<string, unknown>, signal?: AbortSignal) => {
+ const [base, query] = path.split("?");
+ if (command.action === "close") return resources.delete(base, cwd, signal);
+ if (command.action === "kill") return resources.delete(`${base}?force=true`, cwd, signal);
+ const options = new URLSearchParams(query ?? "");
+ const address = { ...at(base), ...(options.get("hint") ? { hint: options.get("hint")! } : {}) } as ComputerAddress;
+ const delivery = options.get("delivery");
+ return resources.perform(address, cwd, JSON.stringify(delivery ? { ...command, delivery_mode: delivery } : command), signal);
+ };
return {
resources, machine, cwd, runner,
/** Where to Search this machine's windows. */
windows: computerSource("machine", name, "windows"),
/** Where to Read this machine's displays. */
displays: computerSource("machine", name, "displays"),
- read: (path: string, views?: string[]) => resources.read(at(path), cwd, views) as Promise<ComputerResult>,
+ // Live checks drive the engine under the computer_* tools directly; a sequence view is computer_record's capture.
+ read: (path: string, views?: string[]) => (views?.some(view => view.startsWith("sequence")) ? resources.sequence(at(path), cwd, views) : resources.read(at(path), cwd, views)) as Promise<ComputerResult>,
search: (path: string, query: string) => resources.search(at(path), cwd, query),
- insert: (path: string, command: unknown, signal?: AbortSignal) => resources.perform(at(path), cwd, JSON.stringify(command), "insert", signal),
+ insert: (path: string, command: unknown, signal?: AbortSignal) => engine(path, command as Record<string, unknown>, signal),
/** Run a Windows PowerShell command on the machine, like a human changing something outside the session. */
powershell: (command: string) => runner.run(WINDOWS_POWERSHELL, ["-NoProfile", "-Command", command]),
/** Ask one window this check opened to close (WM_CLOSE to that HWND); other windows of its process stay open. */
@@ -47,8 +61,8 @@ export async function connectLive(cwd = process.cwd()) {
`Add-Type -Name Close -Namespace PiLive -MemberDefinition '[DllImport("user32.dll")] public static extern bool PostMessage(IntPtr h, uint m, IntPtr w, IntPtr l);'; [void][PiLive.Close]::PostMessage([IntPtr]${hwnd}, 0x10, [IntPtr]::Zero, [IntPtr]::Zero)`]).catch(() => ""),
/** Force-close a process this check opened, for example a modified test document. Only for apps with one process per window. */
kill: (pid: number) => runner.run(WINDOWS_POWERSHELL, ["-NoProfile", "-Command", `Stop-Process -Id ${pid} -Force -ErrorAction SilentlyContinue`]).catch(() => ""),
- write: (path: string, text: string) => resources.perform(at(path), cwd, text, "write"),
- close: () => resources.perform(at("computer:session"), cwd, JSON.stringify({ action: "close" }), "insert").catch(() => undefined),
+ write: (path: string, text: string) => engine(path, { action: "type_text", text }),
+ close: () => resources.delete("computer:session", cwd).catch(() => undefined),
};
}
diff --git a/test/live/semantic-focus.live.ts b/test/live/semantic-focus.live.ts
index b737e1c..2b149a8 100644
--- a/test/live/semantic-focus.live.ts
+++ b/test/live/semantic-focus.live.ts
@@ -83,8 +83,7 @@ test("strict UIA actions keep the sentinel foreground and the target's desktop",
};
const focus = async (window: string) => {
- const state = await live.read(window, ["state"]);
- expect((await live.insert(`computer:window-state/${state.details.window_state_id}`, { action: "focus" })).details.status).toBe("delivered");
+ expect((await live.resources.manageWindow(live.resources.address(window), live.cwd, { action: "focus" })).details.status).toBe("delivered");
await sleep(1000);
};
await focus(sentinel);
diff --git a/test/live/short-addresses.live.ts b/test/live/short-addresses.live.ts
index 88e40c6..a47116d 100644
--- a/test/live/short-addresses.live.ts
+++ b/test/live/short-addresses.live.ts
@@ -1,5 +1,6 @@
import { afterAll, expect, test } from "vitest";
-import { performComputer, readComputer } from "../../src/provider.ts";
+import { computerAction, readComputer } from "../../src/provider.ts";
+import { clickPlan } from "../../src/computer-tools.ts";
import { connectLive, nativeForm, openWindow, WINDOWS_POWERSHELL_PATH } from "./remote.ts";
// LPT-932: on a configured machine, results show short addresses with the machine, and a named window is used by name.
@@ -10,12 +11,15 @@ const read = async (source: string, views?: string[]) => {
const result = await readComputer(live.resources, source, live.cwd, { views }, false);
return result.content[0].type === "text" ? result.content[0].text : "";
};
-const insert = (source: string, command: unknown) => performComputer(live.resources, "insert", source, { path: source, text: JSON.stringify(command) }, live.cwd);
+// What computer_window name and computer_click return: the summary and the script receipt.
+const nameWindow = (source: string, as: string) => computerAction(live.resources, source, "name", () => live.resources.address(source), address => live.resources.name(address, live.cwd, as));
+const click = (source: string, target: string) => computerAction(live.resources, source, "click", () => live.resources.address(source),
+ address => live.resources.input(address, live.cwd, clickPlan({ window: source, target })));
test("a window is read and acted on by its short address and by a name, and the name ends with the window", async () => {
const title = `pi-short-${Date.now()}`;
const long = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, nativeForm(title), title);
- const named = await insert(live.machine, { action: "name", as: "form", title });
+ const named = await nameWindow(long, "form");
expect(named.script.outcome, named.summary).toBe("delivered");
const short = named.script.window as string;
expect(short).toMatch(new RegExp(`^computer:${name}/w\\d+[a-z]{2}$`));
@@ -23,7 +27,7 @@ test("a window is read and acted on by its short address and by a name, and the
expect(view).toMatch(new RegExp(`^Computer observed \\(yolo\\) on ${name}\\nWindow: ${short}\\nObservation: computer:${name}/o\\d+[a-z]{2}$`, "m"));
expect(view).not.toMatch(/computer:(window|observation)\/[wd0-9a-f-]{8}/);
// Act by name: a locator click on the form's counter button.
- const clicked = await insert("computer:form", { action: "click", target: "getByRole('button', { name: 'Count 0' })" });
+ const clicked = await click("computer:form", "getByRole('button', { name: 'Count 0' })");
expect(clicked.script.outcome, clicked.summary).toBe("delivered");
expect(await read("computer:form", ["ui"])).toContain("Count 1");
expect(await read("computer:session")).toMatch(new RegExp(`computer:form → ${short} \\| ${title}`));
@@ -32,7 +36,7 @@ test("a window is read and acted on by its short address and by a name, and the
await live.closeWindow((state.details.target as { window_id: number }).window_id);
for (let i = 0; i < 20 && ((await live.search(live.windows, title)).details.state?.windows as unknown[] | undefined)?.length; i++) await new Promise(resolve => setTimeout(resolve, 500));
await live.search(live.windows, "");
- const gone = await insert("computer:form", { action: "click", target: "getByRole('button')" });
+ const gone = await click("computer:form", "getByRole('button')");
expect(gone.script.outcome).toBe("refused");
expect(gone.summary).toMatch(/No window is named form/);
});
diff --git a/test/remote-machine.test.ts b/test/remote-machine.test.ts
index f06ff24..f59ff20 100644
--- a/test/remote-machine.test.ts
+++ b/test/remote-machine.test.ts
@@ -91,13 +91,13 @@ test("a machine starts on first use; its windows, apps and displays come only fr
const listed = await f.resources.search(address(`${f.machineSource}/windows`), cwd, "window");
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
const view = await f.resources.read(address(computerSource("window", ref)), cwd, ["ui"]);
- await f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, "hello", "write");
+ await f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, JSON.stringify({ action: "type_text", text: "hello" }));
const displays = await f.resources.read(address(`${f.machineSource}/displays`), cwd);
expect((displays!.details.state!.displays as { display_id: string }[]).map(display => display.display_id)).toEqual(["D20"]);
// An app address opens the app on the machine that listed it.
const apps = await f.resources.search(address(`${f.machineSource}/apps`), cwd, "App");
const app = (apps.details.state!.apps as { app_ref: string }[])[0].app_ref;
- await f.resources.perform(address(computerSource("app", app)), cwd, JSON.stringify({ action: "open" }), "insert");
+ await f.resources.perform(address(computerSource("app", app)), cwd, JSON.stringify({ action: "open" }));
// The Write's successor walks the window again; repeated walks are collapsed here.
expect(f.names(f.remoteDriver).filter((name, i, all) => name !== "get_window_state" || all[i - 1] !== "get_window_state")).toEqual(["list_windows", "get_window_state", "type_text", "get_window_state", "list_displays", "list_apps", "list_apps", "list_windows", "launch_app"]);
expect(f.names(f.hostDriver)).toEqual([]);
@@ -111,17 +111,17 @@ test("an unknown machine refuses, and closing a machine releases it, its referen
const listed = await f.resources.search(address(`${f.machineSource}/windows`), cwd, "window");
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
const view = await f.resources.read(address(computerSource("window", ref)), cwd, ["ui"]);
- const sandbox = await f.resources.perform(address(f.machineSource), cwd, JSON.stringify({ action: "open_sandbox" }), "insert");
+ const sandbox = await f.resources.perform(address(f.machineSource), cwd, JSON.stringify({ action: "open_sandbox" }));
expect(sandbox.details.sandbox_ref).toBe("sb-1");
expect(f.openSandbox.mock.calls[0][3]).toBe("lab");
- const closed = await f.resources.perform(address(f.machineSource), cwd, JSON.stringify({ action: "close" }), "insert");
+ const closed = await f.resources.delete(f.machineSource, cwd);
// The report names each released part.
expect(closed.details.status).toBe("closed");
expect(closed.content[0]).toMatchObject({ text: expect.stringMatching(/computer:sandbox\/sb-1: Sandbox[^\n]*destroyed[\s\S]*computer:machine\/lab: /) });
expect(f.sandboxClose).toHaveBeenCalledOnce();
expect(f.machineClose).toHaveBeenCalledOnce();
expect(() => f.resources.descriptor(address(computerSource("sandbox", "sb-1")))).toThrow(/expired/);
- await expect(f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, "late", "write")).rejects.toThrow(/expired/);
+ await expect(f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, JSON.stringify({ action: "type_text", text: "late" }))).rejects.toThrow(/expired/);
// A stale remote window address never reaches this host's desktop.
await expect(f.resources.read(address(computerSource("window", ref)), cwd, ["ui"])).rejects.toThrow(/Unknown or closed window reference[\s\S]*nothing was sent/);
expect(f.names(f.hostDriver)).toEqual([]);
@@ -131,7 +131,7 @@ test("closing the computer session releases every machine, the local one include
const f = await fixture();
await f.resources.search(address(`${f.machineSource}/windows`), cwd, "");
await f.resources.search(address("computer:machine/local/windows"), cwd, "");
- const closed = await f.resources.perform(address("computer:session"), cwd, JSON.stringify({ action: "close" }), "insert");
+ const closed = await f.resources.delete("computer:session", cwd);
expect(f.machineClose).toHaveBeenCalledOnce();
expect(closed.details.status).toBe("closed");
// What the model sees: a session's own close result carries a JSON projection that must not hide later parts.
@@ -155,7 +155,7 @@ test("window sequences and their frames run on the machine that captured them",
const listed = await f.resources.search(address(`${f.machineSource}/windows`), cwd, "window");
expect(listed.details.machine).toBe("lab");
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
- const captured = await f.resources.read(address(computerSource("window", ref)), cwd, ['sequence:{"duration_ms":1}']);
+ const captured = await f.resources.sequence(address(computerSource("window", ref)), cwd, ['sequence:{"duration_ms":1}']);
expect(captured?.details).toMatchObject({ status: "observed", machine: "lab" });
const sequence = address(computerSource("sequence", captured!.details.sequence!.sequence_id));
const frames = await f.resources.read(sequence, cwd, ["frames:endpoints"]);
@@ -172,13 +172,13 @@ test("a machine's primary and listed displays are read and captured on that mach
expect(overview?.details).toMatchObject({ status: "observed", machine: "lab" });
// The display observation keeps its machine.
expect(f.resources.descriptor(address(computerSource("observation", overview!.details.agent_image!.image_id as string)))).toContain(`Source: ${displaySource("lab", "primary")}`);
- const sequence = await f.resources.read(primary, cwd, ['sequence:{"duration_ms":1}']);
+ const sequence = await f.resources.sequence(primary, cwd, ['sequence:{"duration_ms":1}']);
expect(sequence?.details).toMatchObject({ status: "observed", machine: "lab" });
const listed = await f.resources.read(address(`${f.machineSource}/displays`), cwd);
const ref = (listed!.details.state!.displays as { display_ref: string }[])[0].display_ref;
// The listing shows each display with its machine's address.
expect(presentComputerResult(listed!)).toContain(displaySource("lab", ref));
- const selected = await f.resources.read(address(displaySource("lab", ref)), cwd, ['sequence:{"duration_ms":1}']);
+ const selected = await f.resources.sequence(address(displaySource("lab", ref)), cwd, ['sequence:{"duration_ms":1}']);
expect(selected?.details).toMatchObject({ status: "observed", machine: "lab" });
const frames = await f.resources.read(address(computerSource("sequence", selected!.details.sequence!.sequence_id)), cwd, ["frames:last"]);
expect(frames?.details.status).toBe("observed");
diff --git a/test/sandbox.test.ts b/test/sandbox.test.ts
index 3f8a8d5..40220df 100644
--- a/test/sandbox.test.ts
+++ b/test/sandbox.test.ts
@@ -118,7 +118,7 @@ async function routing() {
const close = vi.fn(async () => {});
const sandbox: OpenedSandbox = { ref: "sb-test", session: guest, describe: source => `# Windows Sandbox\nAddress: ${source}`, close };
const resources = new ComputerResources(async () => host, async () => host.close(), async () => sandbox);
- const opened = await resources.perform(address("computer:machine/local"), cwd, JSON.stringify({ action: "open_sandbox" }), "insert");
+ const opened = await resources.perform(address("computer:machine/local"), cwd, JSON.stringify({ action: "open_sandbox" }));
return { hostDriver, guestDriver, resources, opened, close, sandboxSource: computerSource("sandbox", "sb-test") };
}
@@ -130,7 +130,7 @@ test("guest windows are found, read and operated only through the sandbox's own
expect(row.source_identity.domain).toBe("windows-sandbox");
const view = await f.resources.read(address(computerSource("window", row.window_ref)), cwd, ["ui"]);
expect(view?.details.status).toBe("observed");
- await f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, "hello", "write");
+ await f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, JSON.stringify({ action: "type_text", text: "hello" }));
const names = (driver: Driver) => vi.mocked(driver.call).mock.calls.map(([name]) => name);
// The Write's successor walks the window again; repeated walks are collapsed here.
expect(names(f.guestDriver).filter((name, i, all) => name !== "get_window_state" || all[i - 1] !== "get_window_state")).toEqual(["list_windows", "get_window_state", "type_text", "get_window_state"]);
@@ -143,16 +143,16 @@ test("destroying a sandbox retires its address and guest references", async () =
const listed = await f.resources.search(address(f.sandboxSource), cwd, "window");
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
const view = await f.resources.read(address(computerSource("window", ref)), cwd, ["ui"]);
- const closed = await f.resources.perform(address(f.sandboxSource), cwd, JSON.stringify({ action: "close" }), "insert");
+ const closed = await f.resources.delete(f.sandboxSource, cwd);
expect(closed.details).toMatchObject({ status: "closed", sandbox_ref: "sb-test" });
expect(f.close).toHaveBeenCalledOnce();
expect(() => f.resources.descriptor(address(f.sandboxSource))).toThrow(/expired/);
- await expect(f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, "late", "write")).rejects.toThrow(/expired/);
+ await expect(f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, JSON.stringify({ action: "type_text", text: "late" }))).rejects.toThrow(/expired/);
});
test("closing the computer session destroys its sandboxes", async () => {
const f = await routing();
- await f.resources.perform(address("computer:session"), cwd, JSON.stringify({ action: "close" }), "insert");
+ await f.resources.delete("computer:session", cwd);
expect(f.close).toHaveBeenCalledOnce();
expect(() => f.resources.descriptor(address(f.sandboxSource))).toThrow(/expired/);
});
diff --git a/test/setup-command.test.ts b/test/setup-command.test.ts
index 1edc1f9..79f40d3 100644
--- a/test/setup-command.test.ts
+++ b/test/setup-command.test.ts
@@ -98,19 +98,6 @@ test("setup refuses before any command for an unknown machine, another OS or a m
expect(calls).toEqual([]);
});
-test("the agent's setup Insert installs without starting the machine's driver and refuses malformed requests first", async () => {
- const cwd = await project();
- const getSession = vi.fn(), start = vi.fn();
- const resources = new ComputerResources(getSession, async () => { throw new Error("unused"); }, undefined, start);
- const insert = (ref: string, command: unknown) => resources.perform(parseComputerAddress(computerSource("machine", ref))!, cwd, JSON.stringify(command), "insert");
- await expect(insert("local", { action: "setup", part: "flux" })).rejects.toThrow(/setup needs part.*explicitly allowed/);
- await expect(insert("local", { action: "setup", part: "mxc", ssh: "x" })).rejects.toThrow(/belong to setup part machine/);
- await expect(insert("lab", { action: "setup", part: "machine", ssh: "lab-wsl" })).rejects.toThrow(/from computer:machine\/local/);
- await expect(insert("nowhere", { action: "setup", part: "mxc" })).rejects.toThrow(ComputerRequestError);
- expect(getSession).not.toHaveBeenCalled();
- expect(start).not.toHaveBeenCalled();
-});
-
test("/computer-setup shows the usage, reports progress in the status line and the result or failure as a notification", async () => {
const commands = new Map<string, { getArgumentCompletions: (prefix: string) => unknown; handler: (args: string, ctx: unknown) => Promise<void> }>();
const run = vi.fn(async (_request: unknown, _cwd: string, options?: { progress?: (step: string) => void }) => { options?.progress?.("downloading"); return "Installed."; });
@@ -137,5 +124,5 @@ test("browsers use the portable Windows Node from setup first, an installed Node
const installed = "/mnt/c/Program Files/nodejs/node.exe";
expect(await findWindowsNode(undefined, fake([portable, installed]))).toBe(portable);
expect(await findWindowsNode(undefined, fake([installed]))).toBe(installed);
- await expect(findWindowsNode(undefined, fake([]))).rejects.toThrow(/Ask the user.*"part":"browser"/);
+ await expect(findWindowsNode(undefined, fake([]))).rejects.toThrow(/Ask the user.*\/computer-setup browser/);
});
diff --git a/test/short-addresses.test.ts b/test/short-addresses.test.ts
index 426c6f7..ecd66b8 100644
--- a/test/short-addresses.test.ts
+++ b/test/short-addresses.test.ts
@@ -2,7 +2,7 @@ import { expect, test, vi } from "vitest";
import { computerSource } from "../src/computer-address.ts";
import { ComputerRequestError, ComputerResources, type OpenedMachine } from "../src/computer-resources.ts";
import { loadComputerConfig } from "../src/config.ts";
-import { performComputer, readComputer } from "../src/provider.ts";
+import { computerAction, readComputer } from "../src/provider.ts";
import { ShortAddresses } from "../src/short-addresses.ts";
import { ComputerSession, type Driver } from "../src/session.ts";
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
@@ -33,10 +33,11 @@ function fixture() {
const machine: OpenedMachine = { ref: "lab", session: remoteSession, describe: source => source, close: () => remoteSession.close() };
const resources = new ComputerResources(async () => hostSession, async () => hostSession.close(), undefined, async () => machine);
const search = async (source: string, query = "") => (await resources.search(resources.address(source), cwd, query)).details.state!.windows as { window_ref: string; title: string }[];
- const insert = (source: string, command: unknown) => performComputer(resources, "insert", source, { path: source, text: JSON.stringify(command) }, cwd);
+ // computer_window name: the receipt and summary the tool returns.
+ const name = (source: string, as: string) => computerAction(resources, source, "name", () => resources.address(source), address => resources.name(address, cwd, as));
const read = (source: string, views?: string[]) => readComputer(resources, source, cwd, { views }, false);
const text = (read: Awaited<ReturnType<typeof readComputer>>) => read.content[0].type === "text" ? read.content[0].text : "";
- return { resources, host, remote, search, insert, read, text };
+ return { resources, host, remote, search, name, read, text };
}
// LPT-932: the agent sees short addresses with the machine and a run tag, and can name windows.
@@ -52,12 +53,11 @@ test("presented addresses are short, per machine and kind, and resolve back to t
expect(body).toMatch(new RegExp(`^Computer observed \\(yolo\\) on lab\\nWindow: computer:lab/w1${tag}\\nObservation: computer:lab/o1${tag}$`, "m"));
expect(body).not.toMatch(/computer:(window|observation)\//);
expect(f.resources.address(`computer:lab/w1${tag}`)).toMatchObject({ kind: "window", ref: game.window_ref });
- // An observation alias keeps the long form's input options.
- expect(f.resources.address(`computer:lab/o1${tag}?delivery=foreground`)).toMatchObject({ kind: "observation", ref: view.details.computer!.observation_id, delivery: "foreground" });
+ expect(f.resources.address(`computer:lab/o1${tag}`)).toMatchObject({ kind: "observation", ref: view.details.computer!.observation_id });
// The short address works for Read; long addresses still work too.
expect(f.text(await f.read(`computer:lab/w1${tag}`, ["ui"]))).toContain(`Observation: computer:lab/o2${tag}`);
// Receipts for scripts carry the short addresses as well.
- const receipt = await f.insert(`computer:lab/w1${tag}`, { action: "name", as: "game" });
+ const receipt = await f.name(`computer:lab/w1${tag}`, "game");
expect(receipt.script).toMatchObject({ outcome: "delivered", window: `computer:lab/w1${tag}` });
});
@@ -73,19 +73,18 @@ test("an alias from another run, or one never issued, refuses with the reason",
test("names resolve to their window, are unique while the window lives, and stop when it closes", async () => {
const f = fixture();
const [game] = await f.search("computer:machine/lab/windows", "regex:^Game$");
- const named = await f.insert(computerSource("window", game.window_ref), { action: "name", as: "game" });
+ const named = await f.name(computerSource("window", game.window_ref), "game");
expect(named.summary).toMatch(/computer:game now names computer:lab\/w1\w\w \(Game\)/);
expect(f.resources.address("computer:game")).toMatchObject({ kind: "window", ref: game.window_ref });
expect(f.text(await f.read("computer:game", ["ui"]))).toMatch(/^Computer observed \(yolo\) on lab/);
- // Several windows match a title: the name command lists them instead of choosing.
- const several = await f.insert("computer:machine/lab", { action: "name", as: "settings", title: "Game" });
- expect(several.script.outcome).toBe("refused");
- expect(several.summary).toMatch(/2 windows match "Game"[\s\S]*computer:lab\/w1\w\w \| Game\n- computer:lab\/w2\w\w \| Game settings/);
- const settings = await f.insert("computer:machine/lab", { action: "name", as: "settings", title: "settings" });
+ const [other] = await f.search("computer:machine/lab/windows", "settings");
+ const settings = await f.name(computerSource("window", other.window_ref), "settings");
expect(settings.script.outcome).toBe("delivered");
+ // Only windows take names.
+ expect((await f.name("computer:machine/lab", "lab2")).script.outcome).toBe("refused");
// A live window's name is not taken over; bad names refuse.
- expect((await f.insert("computer:settings", { action: "name", as: "game" })).summary).toMatch(/computer:game already names computer:lab\/w1\w\w \(Game\)/);
- for (const bad of ["window", "lab", "Game", "9x", "a b"]) expect((await f.insert("computer:game", { action: "name", as: bad })).script.outcome).toBe("refused");
+ expect((await f.name("computer:settings", "game")).summary).toMatch(/computer:game already names computer:lab\/w1\w\w \(Game\)/);
+ for (const bad of ["window", "lab", "Game", "9x", "a b"]) expect((await f.name("computer:game", bad)).script.outcome).toBe("refused");
const session = f.text(await f.read("computer:session"));
expect(session).toMatch(/Window names[\s\S]*- computer:game → computer:lab\/w1\w\w \| Game\n- computer:settings → computer:lab\/w2\w\w \| Game settings/);
// The window closes: its name disappears from the list and refuses.
diff --git a/test/window-restore.test.ts b/test/window-restore.test.ts
index e77e8f0..2b08e2a 100644
--- a/test/window-restore.test.ts
+++ b/test/window-restore.test.ts
@@ -42,11 +42,10 @@ async function fixture() {
const result = await resources.read(window, "/fixture", ["state"]);
expect(result?.details.status).toBe("observed");
expect(result?.content.some(block => block.type === "image")).toBe(false);
- const source = computerSource("window-state", result!.details.window_state_id!);
- return { result: result!, address: requireComputerAddress(source), source };
+ return result!;
};
- const restore = (address: ReturnType<typeof requireComputerAddress>, signal?: AbortSignal) =>
- resources.perform(address, "/fixture", '{"action":"restore"}', "insert", signal);
+ // computer_window restore: a fresh state sample bounds exactly one restore.
+ const restore = (signal?: AbortSignal) => resources.manageWindow(window, "/fixture", { action: "restore" }, signal);
return { driver, session, resources, window, calls, restores, read, restore,
setLife(value: string | undefined) { life = value; }, setMode(value: ComputerMode) { mode = value; },
setCapability(value: boolean) { capability = value; }, afterStatus(value: () => void) { afterStatus = value; },
@@ -54,62 +53,45 @@ async function fixture() {
outcome(value: Record<string, unknown>, isError = false) { receipt = value; error = isError; } };
}
-test("fresh metadata-only Read authorizes one exact restore without acquiring pixels or UI", async () => {
- const f = await fixture(), view = await f.read();
- const before = f.calls().length;
- expect(f.resources.descriptor(view.address)).toContain("restore");
- expect(f.calls()).toHaveLength(before);
- const result = await f.restore(view.address);
+test("restore samples the window's state and restores exactly that window without pixels or UI", async () => {
+ const f = await fixture();
+ const result = await f.restore();
expect(result.details.status).toBe("delivered");
expect(f.restores()).toEqual([["restore_window", { pid: 42, window_id: 91, state_id: "native-1" }, undefined]]);
expect(f.calls().filter(([name]) => name === "get_window_state")).toHaveLength(0);
- await expect(f.restore(view.address)).rejects.toThrow(/expired|retired/i);
- expect(f.restores()).toHaveLength(1);
+ // Each restore samples its own state; nothing from an earlier call is reused.
+ expect((await f.restore()).details.status).toBe("delivered");
+ expect(f.restores().at(-1)).toEqual(["restore_window", { pid: 42, window_id: 91, state_id: "native-2" }, undefined]);
});
-test("window state is not literal, pixel, token, sequence or caller-supplied identity authority", async () => {
- const f = await fixture(), view = await f.read();
- for (const [tool, value] of [["write", "hello"], ["insert", '{"action":"click","x":1,"y":1}'],
- ["insert", '{"action":"restore","window_id":92}'], ["insert", '{"action":"restore","state_id":"other"}'],
- ["insert", '{"action":"restore","delivery_mode":"foreground"}'], ["insert", '{"action":"sequence","sequence":[]}']] as const)
- await expect(f.resources.perform(view.address, "/fixture", value, tool)).rejects.toThrow();
- expect(f.restores()).toHaveLength(0);
- expect(f.calls().filter(([name]) => name === "get_window_status")).toHaveLength(1);
-});
-
-test("fresh window parts retire old state, and state Read retires old UI authority", async () => {
- const f = await fixture(), first = await f.read(), next = await f.read();
- await expect(f.restore(first.address)).rejects.toThrow(/expired|retired/i);
+test("a state Read only looks: it hands out no state address and keeps the window's observations", async () => {
+ const f = await fixture();
const ui = await f.resources.read(f.window, "/fixture", ["ui"]);
- await expect(f.restore(next.address)).rejects.toThrow(/expired|retired/i);
+ const view = await f.read();
+ expect(view.content.map(block => block.type === "text" ? block.text : "").join("\n")).not.toMatch(/window-state|Insert/);
const observation = requireComputerAddress(computerSource("observation", ui!.details.observation_id!));
- await f.read();
- await expect(f.resources.perform(observation, "/fixture", "text", "write")).rejects.toThrow(/expired|retired/i);
+ expect((await f.resources.read(observation, "/fixture", ["ui"]))?.details.status).toBe("observed");
+ expect(() => requireComputerAddress(`computer:window-state/${view.details.window_state_id}`)).toThrow();
expect(f.restores()).toHaveLength(0);
});
-test.each(["safe", "closed", "reused"] as const)("%s state cannot dispatch or silently refresh restoration", async change => {
- const f = await fixture(), view = await f.read();
+test.each(["safe", "closed", "reused"] as const)("%s window cannot be restored", async change => {
+ const f = await fixture();
if (change === "safe") f.setMode("safe"); else f.setLife(change === "closed" ? undefined : "new-life");
- expect((await f.restore(view.address)).details.status).toBe("refused");
+ expect((await f.restore()).details.status).toBe("refused");
expect(f.restores()).toHaveLength(0);
- expect(f.calls().filter(([name]) => name === "get_window_status")).toHaveLength(1);
- await expect(f.restore(view.address)).rejects.toThrow(/expired|retired/i);
});
-test("a window reused during status acquisition grants no state reference", async () => {
+test("a window reused during status acquisition is not restored", async () => {
const f = await fixture(); f.afterStatus(() => f.setLife("replacement-life"));
- const result = await f.resources.read(f.window, "/fixture", ["state"]);
- expect(result?.details.status).toBe("refused");
- expect(result?.details.window_state_id).toBeUndefined();
+ expect((await f.restore()).details.status).toBe("refused");
expect(f.restores()).toHaveLength(0);
});
-test("cancellation before dispatch consumes the explicit command without native restore", async () => {
- const f = await fixture(), view = await f.read(), controller = new AbortController(); controller.abort();
- expect((await f.restore(view.address, controller.signal)).details.status).toBe("refused");
+test("cancellation before dispatch sends no native restore", async () => {
+ const f = await fixture(), controller = new AbortController(); controller.abort();
+ expect((await f.restore(controller.signal)).details.status).toBe("refused");
expect(f.restores()).toHaveLength(0);
- await expect(f.restore(view.address)).rejects.toThrow(/expired|retired/i);
});
test.each([
@@ -117,48 +99,25 @@ test.each([
{ restore_requested: true, restored: false, activation_attempted: false, activation_succeeded: false,
foreground_at_target: null, effect: "unknown", code: "cancelled" },
])("partial restore/cancellation keeps unknown effects and never replays", async receipt => {
- const f = await fixture(), view = await f.read(); f.outcome(receipt, true);
- const result = await f.restore(view.address);
+ const f = await fixture(); f.outcome(receipt, true);
+ const result = await f.restore();
expect(result.details.status).toBe("unknown");
expect(result.details.state).toMatchObject({ restore_attempted: true, ...receipt });
- await expect(f.restore(view.address)).rejects.toThrow(/expired|retired/i);
expect(f.restores()).toHaveLength(1);
});
test.each([{ window_id: 92 }, { state_id: "wrong-state" }, { foreground_at_target: null },
{ visible: false }, { activation_succeeded: false }, { route: "assisted" }, { effect: "unverifiable" }])(
"malformed success never disguises incomplete restoration", async receipt => {
- const f = await fixture(), view = await f.read(); f.outcome(receipt);
- expect((await f.restore(view.address)).details.status).toBe("unknown");
+ const f = await fixture(); f.outcome(receipt);
+ expect((await f.restore()).details.status).toBe("unknown");
expect(f.restores()).toHaveLength(1);
});
-test("cached state Read does not reacquire or renew restore authority", async () => {
- const f = await fixture(), view = await f.read();
- const cached = await f.resources.read(view.address, "/fixture");
- expect(cached?.details).toEqual(view.result.details);
- const cachedText = cached?.content.find(block => block.type === "text");
- const freshText = view.result.content.find(block => block.type === "text");
- expect(cachedText?.type === "text" && cachedText.text).toMatch(/^Cached native window state\./);
- expect(cachedText?.type === "text" && cachedText.text.slice(cachedText.text.indexOf("\n")))
- .toBe(freshText?.type === "text" && freshText.text.slice(freshText.text.indexOf("\n")));
- expect(f.calls().filter(([name]) => name === "get_window_status")).toHaveLength(1);
- await f.restore(view.address);
- await expect(f.resources.read(view.address, "/fixture")).rejects.toThrow(/expired|retired/i);
-});
-
-test.each(["safe", "reused"] as const)("cached %s state is withheld without reacquisition", async change => {
- const f = await fixture(), view = await f.read();
- if (change === "safe") f.setMode("safe"); else f.setLife("replacement-life");
- expect((await f.resources.read(view.address, "/fixture"))?.details.status).toBe("refused");
- expect(f.calls().filter(([name]) => name === "get_window_status")).toHaveLength(1);
- await expect(f.restore(view.address)).rejects.toThrow(/expired|retired/i);
-});
-
test("post-dispatch reuse preserves delivered native prefix as unknown", async () => {
- const f = await fixture(), view = await f.read();
+ const f = await fixture();
f.afterRestore(() => f.setLife("replacement-life"));
- const result = await f.restore(view.address);
+ const result = await f.restore();
expect(result.details.status).toBe("unknown");
expect(result.details.state).toMatchObject({ restore_requested: true, restored: true, activation_succeeded: true });
expect(f.restores()).toHaveLength(1);
diff --git a/test/window-state-address.test.ts b/test/window-state-address.test.ts
index f01ed28..443e195 100644
--- a/test/window-state-address.test.ts
+++ b/test/window-state-address.test.ts
@@ -1,12 +1,10 @@
import { expect, test } from "vitest";
-import { parseComputerAddress, computerSource } from "../src/computer-address.ts";
+import { parseComputerAddress } from "../src/computer-address.ts";
import { parseComputerViews } from "../src/computer-views.ts";
-test("window-state references have no pointer, hint, delivery or literal Write targeting options", () => {
- const source = computerSource("window-state", "state-1");
- expect(parseComputerAddress(source)).toEqual({ kind: "window-state", ref: "state-1", source });
- for (const suffix of ["?hint=1:a", "?delivery=foreground", "?delay_ms=0", "?window_ref=other", "#1"])
- expect(() => parseComputerAddress(source + suffix)).toThrow();
+test("window-state addresses no longer exist: computer_window samples its own state", () => {
+ for (const source of ["computer:window-state/state-1", "computer:window-state/state-1?hint=1:a"])
+ expect(() => parseComputerAddress(source)).toThrow();
});
test("metadata-only state is an exclusive Read mode, never a capture or detail option", () => {