Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
A running watch reports its state; a stopped watch has no pending slots (LPT-949)
alexshpunt-clanker[bot]ea778226c628
diff --git a/src/provider.ts b/src/provider.tsindex e9380bc..97a4e00 100644--- a/src/provider.ts+++ b/src/provider.ts@@ -163,7 +163,10 @@ export function presentComputerResult(result: ComputerResult, full = false): strif (d.sequence) {const sequence = d.sequence;lines.push(`Sequence: ${computerSource("sequence", sequence.sequence_id)}`,- `Frames: ${sequence.frames.length}; stop=${sequence.stop_reason}; elapsed_ms=${Math.round(sequence.elapsed_ms)}; actual_fps=${sequence.actual_fps}; last_frame_age_ms=${sequence.last_frame_age_ms === null ? null : Math.round(sequence.last_frame_age_ms)}; skipped_slots=${sequence.skipped_slots.length}; pending_slots=${sequence.pending_slots}`);+ // A running watch has no stop reason or slots yet; a watch the agent stopped left no slots pending.+ "running" in sequence+ ? `Frames so far: ${sequence.frames.length}; running; elapsed_ms=${Math.round(sequence.elapsed_ms)}${sequence.actual_fps === null ? "" : `; actual_fps=${sequence.actual_fps}`}`+ : `Frames: ${sequence.frames.length}; stop=${sequence.stop_reason}; elapsed_ms=${Math.round(sequence.elapsed_ms)}; actual_fps=${sequence.actual_fps}; last_frame_age_ms=${sequence.last_frame_age_ms === null ? null : Math.round(sequence.last_frame_age_ms)}; skipped_slots=${sequence.skipped_slots.length}${sequence.stop_reason === "stopped" ? "" : `; pending_slots=${sequence.pending_slots}`}`);}if (d.watch) lines.push(d.watch.running? `Watch: ${d.watch.watching} runs in the background as ${computerSource("sequence", d.watch.sequence_id)}; computer_record watch_stop at that address stops it`diff --git a/src/session.ts b/src/session.tsindex 5445c3e..e8216a2 100644--- a/src/session.ts+++ b/src/session.ts@@ -1,6 +1,6 @@import { randomUUID, createHash } from "node:crypto";import { nativeDisplay, selectedDisplaySource, type NativeDisplay } from "./selected-displays.ts";-import { collectSequence, cropSequenceFrame, frameSummary, selectFrames, sequenceMetadata, SequenceStop, SequenceStore, validateSequence, type FrameSequence, type SequenceOptions, type FrameSelection, type SequenceFrame } from "./temporal-capture.ts";+import { collectSequence, cropSequenceFrame, frameSummary, runningSequenceMetadata, selectFrames, sequenceMetadata, SequenceStop, SequenceStore, validateSequence, type FrameSequence, type SequenceOptions, type FrameSelection, type SequenceFrame } from "./temporal-capture.ts";import { changeLines, differs, frameChanges, framesBetween, sheetFrames, WATCH_LIMITS, WATCH_SHEET_FRAMES, writeWatchFiles, type WatchFiles, type WatchSpan } from "./watch.ts";import type { PixelBox } from "./pixel-diff.ts";import { rm } from "node:fs/promises";@@ -121,7 +121,8 @@ export interface ComputerResult {/** Fresh window capture parts; omitted for non-overview results. */granularity?: "image" | "ui" | "full";captured?: { image: boolean; ui: boolean };- sequence?: ReturnType<typeof sequenceMetadata>;+ /** A finished sequence, or one still collecting (`running`), such as a watch just started. */+ sequence?: ReturnType<typeof sequenceMetadata> | ReturnType<typeof runningSequenceMetadata>;selected_frames?: Omit<SequenceFrame, "image">[];/** A watch this result started or stopped: what it watches, whether it still runs, the actions it noted and, once stopped, its files. */watch?: { sequence_id: string; watching: string; running: boolean; marks: FrameMark[]; changed_frames?: number; files?: WatchFiles };@@ -782,7 +783,7 @@ export class ComputerSession {return this.result("observed", `Watching ${watching} at up to ${options.fps} FPS in the background, ${size.width}x${size.height} frames. ` +`Act on the window as usual: each action's receipt names its span and frames in this watch. Read the sequence with frames:last for the newest frame; computer_record watch_stop at the sequence returns the contact sheet, change statistics and files. ` +`It stops by itself after ${Math.round(options.timeout_ms / 1000)} s or ${WATCH_LIMITS.max_frames} frames, and when the session closes.`,- undefined, { ...this.addressed(entry), sequence: sequenceMetadata(sequence), watch: { sequence_id: sequence.sequence_id, watching, running: true, marks: [] } });+ undefined, { ...this.addressed(entry), sequence: runningSequenceMetadata(sequence, performance.now() - watch.started), watch: { sequence_id: sequence.sequence_id, watching, running: true, marks: [] } });}/** Note an action on a window in each running watch of it: the action's span in the watch's time and the frames captured during it so far. */diff --git a/src/temporal-capture.ts b/src/temporal-capture.tsindex 03c3501..8b42852 100644--- a/src/temporal-capture.ts+++ b/src/temporal-capture.ts@@ -112,7 +112,8 @@ export async function collectSequence(options: SequenceOptions,}sequence.elapsed_ms = clock.now() - started;sequence.last_frame_age_ms = sequence.frames.length ? sequence.elapsed_ms - sequence.frames.at(-1)!.timestamp_ms : null;- sequence.pending_slots = slots - sequence.frames.length - sequence.skipped_slots.length;+ // A stop request ends a watch whose duration is only its safety limit: the slots after it were never due.+ sequence.pending_slots = sequence.stop_reason === "stopped" ? 0 : slots - sequence.frames.length - sequence.skipped_slots.length;sequence.sample_span_ms = sequence.frames.length > 1 ? sequence.frames.at(-1)!.timestamp_ms - sequence.frames[0].timestamp_ms : 0;sequence.actual_fps = sequence.sample_span_ms > 0 ? (sequence.frames.length - 1) * 1000 / sequence.sample_span_ms : null;const intervals = sequence.frames.slice(1).map((frame, i) => frame.timestamp_ms - sequence.frames[i].timestamp_ms);@@ -131,6 +132,19 @@ export function sequenceMetadata(sequence: FrameSequence) {return { ...sequence, frames: sequence.frames.map(({ image: _image, ...frame }) => frame),timestamp_boundary: "native_call_completed", input: "none", retention: "session-local; newest 3 sequences within 48 MiB" };}+/** A sequence still collecting: no stop reason or slot counts yet. */+export type RunningSequenceMetadata = Omit<ReturnType<typeof sequenceMetadata>, "stop_reason" | "pending_slots" | "skipped_slots" | "interval_ms">+ & { running: true; stop_reason?: never; pending_slots?: never; skipped_slots?: never; interval_ms?: never };+/**+ * Metadata of a sequence that is still collecting, such as a running watch, `elapsed_ms` after its start: the frames+ * so far and the rate they reach. A stop reason and slot counts exist only once it ends.+ */+export function runningSequenceMetadata(sequence: FrameSequence, elapsed_ms: number): RunningSequenceMetadata {+ const { stop_reason: _stop, pending_slots: _pending, skipped_slots: _skipped, interval_ms: _intervals, ...metadata } = sequenceMetadata(sequence);+ const frames = sequence.frames, span = frames.length > 1 ? frames.at(-1)!.timestamp_ms - frames[0].timestamp_ms : 0;+ return { ...metadata, running: true as const, elapsed_ms, sample_span_ms: span, actual_fps: span > 0 ? (frames.length - 1) * 1000 / span : null,+ last_frame_age_ms: frames.length ? elapsed_ms - frames.at(-1)!.timestamp_ms : null };+}/** Select an ordered subset; changed means exact byte changes, not a model judgement. */export function selectFrames(sequence: FrameSequence, selection: FrameSelection): SequenceFrame[] {const frames = sequence.frames;