Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

A running watch reports its state; a stopped watch has no pending slots (LPT-949)

alexshpunt-clanker[bot]ea778226c628
diff --git a/src/provider.ts b/src/provider.ts
index e9380bc..97a4e00 100644
--- a/src/provider.ts
+++ b/src/provider.ts
@@ -163,7 +163,10 @@ export function presentComputerResult(result: ComputerResult, full = false): str
if (d.sequence) {
const sequence = d.sequence;
lines.push(`Sequence: ${computerSource("sequence", sequence.sequence_id)}`,
- `Frames: ${sequence.frames.length}; stop=${sequence.stop_reason}; elapsed_ms=${Math.round(sequence.elapsed_ms)}; actual_fps=${sequence.actual_fps}; last_frame_age_ms=${sequence.last_frame_age_ms === null ? null : Math.round(sequence.last_frame_age_ms)}; skipped_slots=${sequence.skipped_slots.length}; pending_slots=${sequence.pending_slots}`);
+ // A running watch has no stop reason or slots yet; a watch the agent stopped left no slots pending.
+ "running" in sequence
+ ? `Frames so far: ${sequence.frames.length}; running; elapsed_ms=${Math.round(sequence.elapsed_ms)}${sequence.actual_fps === null ? "" : `; actual_fps=${sequence.actual_fps}`}`
+ : `Frames: ${sequence.frames.length}; stop=${sequence.stop_reason}; elapsed_ms=${Math.round(sequence.elapsed_ms)}; actual_fps=${sequence.actual_fps}; last_frame_age_ms=${sequence.last_frame_age_ms === null ? null : Math.round(sequence.last_frame_age_ms)}; skipped_slots=${sequence.skipped_slots.length}${sequence.stop_reason === "stopped" ? "" : `; pending_slots=${sequence.pending_slots}`}`);
}
if (d.watch) lines.push(d.watch.running
? `Watch: ${d.watch.watching} runs in the background as ${computerSource("sequence", d.watch.sequence_id)}; computer_record watch_stop at that address stops it`
diff --git a/src/session.ts b/src/session.ts
index 5445c3e..e8216a2 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -1,6 +1,6 @@
import { randomUUID, createHash } from "node:crypto";
import { nativeDisplay, selectedDisplaySource, type NativeDisplay } from "./selected-displays.ts";
-import { collectSequence, cropSequenceFrame, frameSummary, selectFrames, sequenceMetadata, SequenceStop, SequenceStore, validateSequence, type FrameSequence, type SequenceOptions, type FrameSelection, type SequenceFrame } from "./temporal-capture.ts";
+import { collectSequence, cropSequenceFrame, frameSummary, runningSequenceMetadata, selectFrames, sequenceMetadata, SequenceStop, SequenceStore, validateSequence, type FrameSequence, type SequenceOptions, type FrameSelection, type SequenceFrame } from "./temporal-capture.ts";
import { changeLines, differs, frameChanges, framesBetween, sheetFrames, WATCH_LIMITS, WATCH_SHEET_FRAMES, writeWatchFiles, type WatchFiles, type WatchSpan } from "./watch.ts";
import type { PixelBox } from "./pixel-diff.ts";
import { rm } from "node:fs/promises";
@@ -121,7 +121,8 @@ export interface ComputerResult {
/** Fresh window capture parts; omitted for non-overview results. */
granularity?: "image" | "ui" | "full";
captured?: { image: boolean; ui: boolean };
- sequence?: ReturnType<typeof sequenceMetadata>;
+ /** A finished sequence, or one still collecting (`running`), such as a watch just started. */
+ sequence?: ReturnType<typeof sequenceMetadata> | ReturnType<typeof runningSequenceMetadata>;
selected_frames?: Omit<SequenceFrame, "image">[];
/** A watch this result started or stopped: what it watches, whether it still runs, the actions it noted and, once stopped, its files. */
watch?: { sequence_id: string; watching: string; running: boolean; marks: FrameMark[]; changed_frames?: number; files?: WatchFiles };
@@ -782,7 +783,7 @@ export class ComputerSession {
return this.result("observed", `Watching ${watching} at up to ${options.fps} FPS in the background, ${size.width}x${size.height} frames. ` +
`Act on the window as usual: each action's receipt names its span and frames in this watch. Read the sequence with frames:last for the newest frame; computer_record watch_stop at the sequence returns the contact sheet, change statistics and files. ` +
`It stops by itself after ${Math.round(options.timeout_ms / 1000)} s or ${WATCH_LIMITS.max_frames} frames, and when the session closes.`,
- undefined, { ...this.addressed(entry), sequence: sequenceMetadata(sequence), watch: { sequence_id: sequence.sequence_id, watching, running: true, marks: [] } });
+ undefined, { ...this.addressed(entry), sequence: runningSequenceMetadata(sequence, performance.now() - watch.started), watch: { sequence_id: sequence.sequence_id, watching, running: true, marks: [] } });
}
/** Note an action on a window in each running watch of it: the action's span in the watch's time and the frames captured during it so far. */
diff --git a/src/temporal-capture.ts b/src/temporal-capture.ts
index 03c3501..8b42852 100644
--- a/src/temporal-capture.ts
+++ b/src/temporal-capture.ts
@@ -112,7 +112,8 @@ export async function collectSequence(options: SequenceOptions,
}
sequence.elapsed_ms = clock.now() - started;
sequence.last_frame_age_ms = sequence.frames.length ? sequence.elapsed_ms - sequence.frames.at(-1)!.timestamp_ms : null;
- sequence.pending_slots = slots - sequence.frames.length - sequence.skipped_slots.length;
+ // A stop request ends a watch whose duration is only its safety limit: the slots after it were never due.
+ sequence.pending_slots = sequence.stop_reason === "stopped" ? 0 : slots - sequence.frames.length - sequence.skipped_slots.length;
sequence.sample_span_ms = sequence.frames.length > 1 ? sequence.frames.at(-1)!.timestamp_ms - sequence.frames[0].timestamp_ms : 0;
sequence.actual_fps = sequence.sample_span_ms > 0 ? (sequence.frames.length - 1) * 1000 / sequence.sample_span_ms : null;
const intervals = sequence.frames.slice(1).map((frame, i) => frame.timestamp_ms - sequence.frames[i].timestamp_ms);
@@ -131,6 +132,19 @@ export function sequenceMetadata(sequence: FrameSequence) {
return { ...sequence, frames: sequence.frames.map(({ image: _image, ...frame }) => frame),
timestamp_boundary: "native_call_completed", input: "none", retention: "session-local; newest 3 sequences within 48 MiB" };
}
+/** A sequence still collecting: no stop reason or slot counts yet. */
+export type RunningSequenceMetadata = Omit<ReturnType<typeof sequenceMetadata>, "stop_reason" | "pending_slots" | "skipped_slots" | "interval_ms">
+ & { running: true; stop_reason?: never; pending_slots?: never; skipped_slots?: never; interval_ms?: never };
+/**
+ * Metadata of a sequence that is still collecting, such as a running watch, `elapsed_ms` after its start: the frames
+ * so far and the rate they reach. A stop reason and slot counts exist only once it ends.
+ */
+export function runningSequenceMetadata(sequence: FrameSequence, elapsed_ms: number): RunningSequenceMetadata {
+ const { stop_reason: _stop, pending_slots: _pending, skipped_slots: _skipped, interval_ms: _intervals, ...metadata } = sequenceMetadata(sequence);
+ const frames = sequence.frames, span = frames.length > 1 ? frames.at(-1)!.timestamp_ms - frames[0].timestamp_ms : 0;
+ return { ...metadata, running: true as const, elapsed_ms, sample_span_ms: span, actual_fps: span > 0 ? (frames.length - 1) * 1000 / span : null,
+ last_frame_age_ms: frames.length ? elapsed_ms - frames.at(-1)!.timestamp_ms : null };
+}
/** Select an ordered subset; changed means exact byte changes, not a model judgement. */
export function selectFrames(sequence: FrameSequence, selection: FrameSelection): SequenceFrame[] {
const frames = sequence.frames;