Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

fix: recording receipts show where the file was fetched and name the typed tools (LPT-934)

alexshpunt-clanker[bot]70c9b4f2a280
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index a92b255..1944b5b 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -152,6 +152,14 @@ async function fileCommand(session: ComputerSession, command: Record<string, unk
return resolve(cwd, value);
};
const done = (text: string, state: Record<string, unknown>): ComputerResult => ({ content: [{ type: "text", text }], details: { status: "delivered", mode, state } });
+ // A session result's text is its summary, then its native JSON; added lines join the summary so every receipt shows them.
+ const said = (result: ComputerResult, ...lines: string[]): ComputerResult => {
+ const [first, ...rest] = result.content;
+ if (first?.type !== "text") return { ...result, content: [...lines.map(line => ({ type: "text" as const, text: line })), ...result.content] };
+ const cut = first.text.indexOf("\n{");
+ const summary = cut < 0 ? first.text : first.text.slice(0, cut);
+ return { ...result, content: [{ ...first, text: [summary, ...lines].join("\n") + (cut < 0 ? "" : first.text.slice(cut)) }, ...rest] };
+ };
try {
if (command.action === "clipboard_read") {
only(command, ["action", "image"]);
@@ -181,7 +189,7 @@ async function fileCommand(session: ComputerSession, command: Record<string, unk
const bytes = await host.get(String(stopped.details.state!.name));
await mkdir(dirname(to), { recursive: true });
await writeFile(to, bytes);
- return { ...stopped, content: [...stopped.content, { type: "text", text: `Fetched it (${bytes.length} bytes) into ${to}.` }] };
+ return said(stopped, `Fetched it (${bytes.length} bytes) into ${to}.`);
}
if (command.action === "audio_record") {
only(command, ["action", "seconds", "to", "transcribe"]);
@@ -196,11 +204,11 @@ async function fileCommand(session: ComputerSession, command: Record<string, unk
const wav = to ?? resolve(cwd, ".tmp", `pi-${name}`);
await mkdir(dirname(wav), { recursive: true });
await writeFile(wav, bytes);
- const lines = [...recorded.content, ...(to ? [{ type: "text" as const, text: `Fetched it (${bytes.length} bytes) into ${to}.` }] : [])];
- if (!command.transcribe) return { ...recorded, content: lines };
+ const fetched = to ? [`Fetched it (${bytes.length} bytes) into ${to}.`] : [];
+ if (!command.transcribe) return said(recorded, ...fetched);
const transcript = await transcribe(wav).finally(() => to ? undefined : rm(wav, { force: true }));
- return { ...recorded, content: [...lines, { type: "text", text: `Transcript (faster-whisper on this machine; can be wrong): ${transcript.text || "(no speech)"}` }],
- details: { ...recorded.details, state: { ...recorded.details.state, transcript } } };
+ const heard = said(recorded, ...fetched, `Transcript (faster-whisper on this machine; can be wrong): ${transcript.text || "(no speech)"}`);
+ return { ...heard, details: { ...heard.details, state: { ...heard.details.state, transcript } } };
}
if (command.action === "files") {
only(command, ["action"]);
diff --git a/src/session.ts b/src/session.ts
index 107b715..14acf09 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -749,7 +749,7 @@ export class ComputerSession {
}
const timer = setTimeout(() => { void this.recordStop(); }, seconds * 1000);
this.recording = { timer, started: Date.now() };
- return this.result("delivered", `Recording the main display for up to ${seconds} s. Insert record_stop to finish earlier and fetch the video.`, state);
+ return this.result("delivered", `Recording the main display for up to ${seconds} s. Stop it earlier with computer_record video_stop (or Insert record_stop) to fetch the video.`, state);
});
}
@@ -769,7 +769,7 @@ export class ComputerSession {
recordStop(): Promise<ComputerResult> {
return this.queue(async () => {
const current = this.recording;
- if (!current || !this.host) return this.result("refused", "No recording is running, or it already stopped at its limit; Insert files to see saved videos.");
+ if (!current || !this.host) return this.result("refused", "No recording is running, or it already stopped at its limit; list the session folder with computer_files (or Insert files) to see saved videos.");
clearTimeout(current.timer);
this.recording = undefined;
const native = await this.driver.call("stop_recording", {});
diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.ts
index 2e1386e..bc5578d 100644
--- a/test/computer-tools.test.ts
+++ b/test/computer-tools.test.ts
@@ -3,7 +3,10 @@ import { requireComputerAddress, ComputerResources } from "../src/computer-resou
import { clipboardCommand, filesCommand, openCommand, recordCommand, waitCommand, windowCommand } from "../src/computer-tools.ts";
import type { WebBrowser, WebLauncher } from "../src/web-browser.ts";
import type { MachineHost } from "../src/machine-host.ts";
-import { deleteComputer } from "../src/provider.ts";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { computerAction, deleteComputer } from "../src/provider.ts";
import { ComputerSession, type ComputerMode, type Driver } from "../src/session.ts";
const cwd = "/synthetic-computer-tools";
@@ -169,3 +172,32 @@ test("Delete reports a window that stays open and refuses what it cannot close",
const session = await kept.remove("computer:session");
expect(session.data).toMatchObject({ ok: true });
});
+
+test("computer_record video_stop says where it fetched the video in the summary the model sees", async () => {
+ const driver: Driver = {
+ identity: vi.fn(async () => undefined) as Driver["identity"], close: vi.fn(async () => {}),
+ call: vi.fn<Driver["call"]>(async name => {
+ if (name === "start_recording") return { content: [{ type: "text", text: "{}" }], structuredContent: { video_active: true } };
+ if (name === "stop_recording") return { content: [{ type: "text", text: "{}" }], structuredContent: { video_active: false } };
+ throw new Error(`Unexpected native call ${name}`);
+ }),
+ };
+ const video = Buffer.from("mp4 bytes");
+ const host = { files: async () => ({ windows: "C:\\pi", wsl: "/mnt/c/pi" }), rename: vi.fn(async () => {}), get: vi.fn(async () => video), removeFiles: async () => {} };
+ const session = new ComputerSession(driver, () => "yolo", undefined, undefined, undefined, undefined, host as unknown as MachineHost);
+ const resources = new ComputerResources(async () => session, async () => session.close());
+ const workspace = await mkdtemp(join(tmpdir(), "pi-record-"));
+ const run = (command: Record<string, unknown>) => computerAction(resources, "computer:machine/local", String(command.action), () => at("computer:machine/local"),
+ address => resources.perform(address, workspace, JSON.stringify(command), "insert"));
+ try {
+ expect((await run({ action: "record_start", seconds: 30 })).summary).toContain("Stop it earlier with computer_record video_stop");
+ const stopped = await run({ action: "record_stop", to: "out/run.mp4" });
+ expect(stopped.summary).toContain("as recording-1.mp4 in the session folder.");
+ expect(stopped.summary).toContain(`Fetched it (${video.length} bytes) into ${join(workspace, "out/run.mp4")}.`);
+ expect(await readFile(join(workspace, "out/run.mp4"))).toEqual(video);
+ expect((await run({ action: "record_stop" })).summary).toContain("list the session folder with computer_files");
+ } finally {
+ await session.close();
+ await rm(workspace, { recursive: true, force: true });
+ }
+});
\ No newline at end of file