Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
fix: recording receipts show where the file was fetched and name the typed tools (LPT-934)
alexshpunt-clanker[bot]70c9b4f2a280
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex a92b255..1944b5b 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -152,6 +152,14 @@ async function fileCommand(session: ComputerSession, command: Record<string, unkreturn resolve(cwd, value);};const done = (text: string, state: Record<string, unknown>): ComputerResult => ({ content: [{ type: "text", text }], details: { status: "delivered", mode, state } });+ // A session result's text is its summary, then its native JSON; added lines join the summary so every receipt shows them.+ const said = (result: ComputerResult, ...lines: string[]): ComputerResult => {+ const [first, ...rest] = result.content;+ if (first?.type !== "text") return { ...result, content: [...lines.map(line => ({ type: "text" as const, text: line })), ...result.content] };+ const cut = first.text.indexOf("\n{");+ const summary = cut < 0 ? first.text : first.text.slice(0, cut);+ return { ...result, content: [{ ...first, text: [summary, ...lines].join("\n") + (cut < 0 ? "" : first.text.slice(cut)) }, ...rest] };+ };try {if (command.action === "clipboard_read") {only(command, ["action", "image"]);@@ -181,7 +189,7 @@ async function fileCommand(session: ComputerSession, command: Record<string, unkconst bytes = await host.get(String(stopped.details.state!.name));await mkdir(dirname(to), { recursive: true });await writeFile(to, bytes);- return { ...stopped, content: [...stopped.content, { type: "text", text: `Fetched it (${bytes.length} bytes) into ${to}.` }] };+ return said(stopped, `Fetched it (${bytes.length} bytes) into ${to}.`);}if (command.action === "audio_record") {only(command, ["action", "seconds", "to", "transcribe"]);@@ -196,11 +204,11 @@ async function fileCommand(session: ComputerSession, command: Record<string, unkconst wav = to ?? resolve(cwd, ".tmp", `pi-${name}`);await mkdir(dirname(wav), { recursive: true });await writeFile(wav, bytes);- const lines = [...recorded.content, ...(to ? [{ type: "text" as const, text: `Fetched it (${bytes.length} bytes) into ${to}.` }] : [])];- if (!command.transcribe) return { ...recorded, content: lines };+ const fetched = to ? [`Fetched it (${bytes.length} bytes) into ${to}.`] : [];+ if (!command.transcribe) return said(recorded, ...fetched);const transcript = await transcribe(wav).finally(() => to ? undefined : rm(wav, { force: true }));- return { ...recorded, content: [...lines, { type: "text", text: `Transcript (faster-whisper on this machine; can be wrong): ${transcript.text || "(no speech)"}` }],- details: { ...recorded.details, state: { ...recorded.details.state, transcript } } };+ const heard = said(recorded, ...fetched, `Transcript (faster-whisper on this machine; can be wrong): ${transcript.text || "(no speech)"}`);+ return { ...heard, details: { ...heard.details, state: { ...heard.details.state, transcript } } };}if (command.action === "files") {only(command, ["action"]);diff --git a/src/session.ts b/src/session.tsindex 107b715..14acf09 100644--- a/src/session.ts+++ b/src/session.ts@@ -749,7 +749,7 @@ export class ComputerSession {}const timer = setTimeout(() => { void this.recordStop(); }, seconds * 1000);this.recording = { timer, started: Date.now() };- return this.result("delivered", `Recording the main display for up to ${seconds} s. Insert record_stop to finish earlier and fetch the video.`, state);+ return this.result("delivered", `Recording the main display for up to ${seconds} s. Stop it earlier with computer_record video_stop (or Insert record_stop) to fetch the video.`, state);});}@@ -769,7 +769,7 @@ export class ComputerSession {recordStop(): Promise<ComputerResult> {return this.queue(async () => {const current = this.recording;- if (!current || !this.host) return this.result("refused", "No recording is running, or it already stopped at its limit; Insert files to see saved videos.");+ if (!current || !this.host) return this.result("refused", "No recording is running, or it already stopped at its limit; list the session folder with computer_files (or Insert files) to see saved videos.");clearTimeout(current.timer);this.recording = undefined;const native = await this.driver.call("stop_recording", {});diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.tsindex 2e1386e..bc5578d 100644--- a/test/computer-tools.test.ts+++ b/test/computer-tools.test.ts@@ -3,7 +3,10 @@ import { requireComputerAddress, ComputerResources } from "../src/computer-resouimport { clipboardCommand, filesCommand, openCommand, recordCommand, waitCommand, windowCommand } from "../src/computer-tools.ts";import type { WebBrowser, WebLauncher } from "../src/web-browser.ts";import type { MachineHost } from "../src/machine-host.ts";-import { deleteComputer } from "../src/provider.ts";+import { mkdtemp, readFile, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { computerAction, deleteComputer } from "../src/provider.ts";import { ComputerSession, type ComputerMode, type Driver } from "../src/session.ts";const cwd = "/synthetic-computer-tools";@@ -169,3 +172,32 @@ test("Delete reports a window that stays open and refuses what it cannot close",const session = await kept.remove("computer:session");expect(session.data).toMatchObject({ ok: true });});++test("computer_record video_stop says where it fetched the video in the summary the model sees", async () => {+ const driver: Driver = {+ identity: vi.fn(async () => undefined) as Driver["identity"], close: vi.fn(async () => {}),+ call: vi.fn<Driver["call"]>(async name => {+ if (name === "start_recording") return { content: [{ type: "text", text: "{}" }], structuredContent: { video_active: true } };+ if (name === "stop_recording") return { content: [{ type: "text", text: "{}" }], structuredContent: { video_active: false } };+ throw new Error(`Unexpected native call ${name}`);+ }),+ };+ const video = Buffer.from("mp4 bytes");+ const host = { files: async () => ({ windows: "C:\\pi", wsl: "/mnt/c/pi" }), rename: vi.fn(async () => {}), get: vi.fn(async () => video), removeFiles: async () => {} };+ const session = new ComputerSession(driver, () => "yolo", undefined, undefined, undefined, undefined, host as unknown as MachineHost);+ const resources = new ComputerResources(async () => session, async () => session.close());+ const workspace = await mkdtemp(join(tmpdir(), "pi-record-"));+ const run = (command: Record<string, unknown>) => computerAction(resources, "computer:machine/local", String(command.action), () => at("computer:machine/local"),+ address => resources.perform(address, workspace, JSON.stringify(command), "insert"));+ try {+ expect((await run({ action: "record_start", seconds: 30 })).summary).toContain("Stop it earlier with computer_record video_stop");+ const stopped = await run({ action: "record_stop", to: "out/run.mp4" });+ expect(stopped.summary).toContain("as recording-1.mp4 in the session folder.");+ expect(stopped.summary).toContain(`Fetched it (${video.length} bytes) into ${join(workspace, "out/run.mp4")}.`);+ expect(await readFile(join(workspace, "out/run.mp4"))).toEqual(video);+ expect((await run({ action: "record_stop" })).summary).toContain("list the session folder with computer_files");+ } finally {+ await session.close();+ await rm(workspace, { recursive: true, force: true });+ }+});\ No newline at end of file