Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

[RSI, feature] trace: tool-call lifecycle events in the session trace (started, heartbeat, elapsed-on-abort) (#3689)

Seth Kartenda4ab9978b97
diff --git a/crates/pa-agent/src/agent.rs b/crates/pa-agent/src/agent.rs
index d60b861cfd..eb859b3f04 100644
--- a/crates/pa-agent/src/agent.rs
+++ b/crates/pa-agent/src/agent.rs
@@ -17,7 +17,7 @@ use crate::agent_loop::{
use crate::stream::StreamFn;
use crate::types::{
AgentEvent, AgentMessage, AgentTool, ImageContent, Model, ThinkingLevel, ToolExecutionMode,
- Usage,
+ ToolProgressCadence, Usage,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -102,6 +102,9 @@ pub struct AgentOptions {
pub follow_up_mode: Option<QueueMode>,
pub session_id: Option<String>,
pub tool_execution: Option<ToolExecutionMode>,
+ /// Heartbeat cadence for in-flight tool calls; `None` keeps the 60s
+ /// default tick.
+ pub tool_progress: Option<ToolProgressCadence>,
}
struct MutableAgentState {
@@ -345,6 +348,7 @@ struct AgentInner {
model_override: Mutex<Option<AgentModelOverride>>,
session_id: Option<String>,
tool_execution: ToolExecutionMode,
+ tool_progress: ToolProgressCadence,
}
impl AgentInner {
@@ -406,7 +410,8 @@ impl AgentInner {
}
AgentEvent::AgentStart
| AgentEvent::TurnStart
- | AgentEvent::ToolExecutionUpdate { .. } => {}
+ | AgentEvent::ToolExecutionUpdate { .. }
+ | AgentEvent::ToolExecutionProgress { .. } => {}
}
let Some(signal) = self.current_signal() else {
@@ -568,6 +573,7 @@ impl AgentInner {
config.get_follow_up_messages = Some(follow_up);
config.get_continuation_messages = continuation;
config.tool_execution = self.tool_execution;
+ config.tool_progress = self.tool_progress;
config.before_tool_call.clone_from(&self.before_tool_call);
config.after_tool_call.clone_from(&self.after_tool_call);
config
@@ -797,6 +803,7 @@ impl Agent {
tool_execution: options
.tool_execution
.unwrap_or(ToolExecutionMode::Parallel),
+ tool_progress: options.tool_progress.unwrap_or_default(),
});
Agent { inner }
}
diff --git a/crates/pa-agent/src/agent_loop/mod.rs b/crates/pa-agent/src/agent_loop/mod.rs
index 86cc0985e7..a503a73eb6 100644
--- a/crates/pa-agent/src/agent_loop/mod.rs
+++ b/crates/pa-agent/src/agent_loop/mod.rs
@@ -109,6 +109,9 @@ pub struct AgentLoopConfig {
pub get_follow_up_messages: Option<PollMessagesFn>,
pub get_continuation_messages: Option<GetContinuationMessagesFn>,
pub tool_execution: ToolExecutionMode,
+ /// Heartbeat cadence for in-flight tool calls (start-side trace
+ /// visibility for long calls); defaults to a 60s tick.
+ pub tool_progress: crate::types::ToolProgressCadence,
pub before_tool_call: Option<BeforeToolCallFn>,
pub after_tool_call: Option<AfterToolCallFn>,
}
@@ -135,6 +138,7 @@ impl AgentLoopConfig {
get_follow_up_messages: None,
get_continuation_messages: None,
tool_execution: ToolExecutionMode::Parallel,
+ tool_progress: crate::types::ToolProgressCadence::default(),
before_tool_call: None,
after_tool_call: None,
}
diff --git a/crates/pa-agent/src/agent_loop/tool_call.rs b/crates/pa-agent/src/agent_loop/tool_call.rs
index 13ec058a29..5ce54ae75a 100644
--- a/crates/pa-agent/src/agent_loop/tool_call.rs
+++ b/crates/pa-agent/src/agent_loop/tool_call.rs
@@ -7,7 +7,8 @@ use std::sync::Arc;
use crate::abort::{is_abort_error, AbortSignal};
use crate::types::{
AfterToolCallContext, AgentContext, AgentEvent, AgentMessage, AgentTool, AgentToolResult,
- AgentToolUpdateCallback, AssistantMessage, BeforeToolCallContext, ToolCall, ToolResultMessage,
+ AgentToolUpdateCallback, AssistantMessage, BeforeToolCallContext, ToolCall,
+ ToolProgressCadence, ToolResultMessage,
};
use super::abort::race_with_abort;
@@ -114,6 +115,7 @@ pub(crate) struct ExecutedToolCallOutcome {
/// the tool fails (or aborts, with the TS message "Tool execution aborted").
pub(crate) async fn execute_prepared_tool_call(
prepared: &PreparedToolCall,
+ progress: ToolProgressCadence,
signal: Option<&AbortSignal>,
emit: &AgentEventSink,
) -> ExecutedToolCallOutcome {
@@ -125,6 +127,19 @@ pub(crate) async fn execute_prepared_tool_call(
let tool_name = prepared.tool.name().to_string();
let args = prepared.tool_call.arguments.clone();
let sink = Arc::clone(emit);
+ let started_at = std::time::Instant::now();
+ // Lifecycle heartbeat: quiet until `progress.after_ms`, then one
+ // `ToolExecutionProgress` event every `progress.every_ms` while the call
+ // still runs. The guard aborts the task on EVERY exit path — return,
+ // panic, and cancellation of this frame — so the unbounded tick loop
+ // never leaks as a detached task.
+ let mut heartbeat = HeartbeatGuard(Some(spawn_tool_heartbeat(
+ Arc::clone(emit),
+ tool_call_id.clone(),
+ tool_name.clone(),
+ started_at,
+ progress,
+ )));
let signal_for_updates = signal.cloned().unwrap_or_default();
let emit_updates = tokio::spawn(async move {
while let Some(partial_result) = update_rx.recv().await {
@@ -161,41 +176,22 @@ pub(crate) async fn execute_prepared_tool_call(
signal,
)
.await;
+ heartbeat.stop();
+ let elapsed_ms = elapsed_ms_since(started_at);
accepting_updates.store(false, std::sync::atomic::Ordering::SeqCst);
drop(update_tx);
match execute_result {
Ok(result) => {
- match race_with_abort(
- async {
- emit_updates.await.map_err(|error| {
- anyhow::anyhow!("Tool update emitter task failed: {error}")
- })
- },
- signal,
- )
- .await
- {
- Ok(update_result) => {
- if let Err(error) = update_result {
- // Success path: a failing update emitter mirrors a
- // rejecting updateEvents promise in TS.
- return ExecutedToolCallOutcome {
- result: error_tool_result(signal, &error),
- is_error: true,
- };
- }
- }
- Err(error) => {
- let aborted = signal.is_some_and(AbortSignal::is_aborted);
- if !(aborted && is_abort_error(&error)) {
- return ExecutedToolCallOutcome {
- result: error_tool_result(signal, &error),
- is_error: true,
- };
- }
- }
+ // Success path: a failing update emitter mirrors a rejecting
+ // updateEvents promise in TS; an aborted drain (the call itself
+ // succeeded) settles cleanly.
+ if let Err(error) = settle_success_drain(emit_updates, signal).await {
+ return ExecutedToolCallOutcome {
+ result: error_tool_result(signal, &error, elapsed_ms),
+ is_error: true,
+ };
}
ExecutedToolCallOutcome {
result,
@@ -216,19 +212,113 @@ pub(crate) async fn execute_prepared_tool_call(
)
.await;
ExecutedToolCallOutcome {
- result: error_tool_result(signal, &error),
+ result: error_tool_result(signal, &error, elapsed_ms),
is_error: true,
}
}
}
}
-fn error_tool_result(signal: Option<&AbortSignal>, error: &anyhow::Error) -> AgentToolResult {
- if signal.is_some_and(AbortSignal::is_aborted) {
+/// Aborts the heartbeat task on every exit path. Dropping the bare
+/// `JoinHandle` only detaches the unbounded tick task: on a panic or a
+/// cancellation (a dropped `execute_prepared_tool_call` future) the explicit
+/// `stop()` never runs, and the detached task would keep emitting
+/// `ToolExecutionProgress` — and pin the event sink — forever.
+struct HeartbeatGuard(Option<tokio::task::JoinHandle<()>>);
+
+impl HeartbeatGuard {
+ /// Stop the heartbeat on the normal path (ticks stop before the
+ /// drain/result emission); `drop` covers panic and cancellation.
+ fn stop(&mut self) {
+ if let Some(handle) = self.0.take() {
+ handle.abort();
+ }
+ }
+}
+
+impl Drop for HeartbeatGuard {
+ fn drop(&mut self) {
+ self.stop();
+ }
+}
+
+/// Drain the update emitter on the success path, racing the drain against
+/// abort: an aborted drain settles cleanly, any other failure surfaces as
+/// the call's error.
+async fn settle_success_drain(
+ emit_updates: tokio::task::JoinHandle<anyhow::Result<()>>,
+ signal: Option<&AbortSignal>,
+) -> anyhow::Result<()> {
+ match race_with_abort(
+ async {
+ emit_updates
+ .await
+ .map_err(|error| anyhow::anyhow!("Tool update emitter task failed: {error}"))
+ },
+ signal,
+ )
+ .await
+ {
+ Ok(update_result) => update_result,
+ Err(error) => {
+ let aborted = signal.is_some_and(AbortSignal::is_aborted);
+ if aborted && is_abort_error(&error) {
+ Ok(())
+ } else {
+ Err(error)
+ }
+ }
+ }
+}
+
+/// Elapsed milliseconds since a call started (millis fit `u64`; the
+/// saturating conversion only guards the type-level truncation lint).
+fn elapsed_ms_since(started_at: std::time::Instant) -> u64 {
+ u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX)
+}
+
+/// The heartbeat task behind the `ToolExecutionProgress` events: quiet until
+/// `progress.after_ms`, then one event every `progress.every_ms` while the
+/// call runs; aborted by the caller once the execution race settles.
+fn spawn_tool_heartbeat(
+ sink: AgentEventSink,
+ tool_call_id: String,
+ tool_name: String,
+ started_at: std::time::Instant,
+ progress: ToolProgressCadence,
+) -> tokio::task::JoinHandle<()> {
+ tokio::spawn(async move {
+ tokio::time::sleep(std::time::Duration::from_millis(progress.after_ms)).await;
+ loop {
+ let _ = sink(AgentEvent::ToolExecutionProgress {
+ tool_call_id: tool_call_id.clone(),
+ tool_name: tool_name.clone(),
+ elapsed_ms: elapsed_ms_since(started_at),
+ })
+ .await;
+ tokio::time::sleep(std::time::Duration::from_millis(progress.every_ms)).await;
+ }
+ })
+}
+
+fn error_tool_result(
+ signal: Option<&AbortSignal>,
+ error: &anyhow::Error,
+ elapsed_ms: u64,
+) -> AgentToolResult {
+ let aborted = signal.is_some_and(AbortSignal::is_aborted);
+ let mut result = if aborted {
AgentToolResult::error("Tool execution aborted")
} else {
AgentToolResult::error(format!("{error:#}"))
- }
+ };
+ // The aborted/killed completion carries the elapsed duration: without
+ // it the trace shows an aborted call but never how long it ran.
+ result.details = serde_json::json!({
+ "status": if aborted { "aborted" } else { "error" },
+ "durationMs": elapsed_ms,
+ });
+ result
}
/// Apply the `afterToolCall` overrides.
@@ -335,3 +425,98 @@ pub(crate) async fn emit_tool_result_message(
})
.await
}
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::Mutex;
+
+ struct NeverTool;
+
+ impl AgentTool for NeverTool {
+ fn name(&self) -> &'static str {
+ "never"
+ }
+ fn description(&self) -> &'static str {
+ "never finishes on its own"
+ }
+ fn parameters(&self) -> &serde_json::Value {
+ static PARAMETERS: std::sync::OnceLock<serde_json::Value> = std::sync::OnceLock::new();
+ PARAMETERS.get_or_init(|| serde_json::json!({ "type": "object" }))
+ }
+ fn execute(
+ self: Arc<Self>,
+ _tool_call_id: String,
+ _params: serde_json::Value,
+ _signal: crate::abort::AbortSignal,
+ _on_update: AgentToolUpdateCallback,
+ ) -> crate::BoxFut<'static, anyhow::Result<AgentToolResult>> {
+ Box::pin(std::future::pending())
+ }
+ }
+
+ /// Cancelling the call frame (a dropped `execute_prepared_tool_call`
+ /// future) must stop the heartbeat task: a detached tick loop would
+ /// keep emitting `ToolExecutionProgress` — and pin the event sink —
+ /// after the run that owned it is gone.
+ #[tokio::test]
+ async fn cancelling_the_call_frame_stops_the_heartbeat() {
+ let ticks: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));
+ let sink: AgentEventSink = {
+ let ticks = Arc::clone(&ticks);
+ Arc::new(move |event| {
+ let ticks = Arc::clone(&ticks);
+ Box::pin(async move {
+ if matches!(event, AgentEvent::ToolExecutionProgress { .. }) {
+ *ticks.lock().unwrap() += 1;
+ }
+ Ok(())
+ })
+ })
+ };
+ let prepared = PreparedToolCall {
+ tool_call: ToolCall {
+ id: "call-1".to_string(),
+ name: "never".to_string(),
+ arguments: serde_json::json!({}),
+ thought_signature: None,
+ },
+ tool: Arc::new(NeverTool),
+ args: serde_json::json!({}),
+ };
+ let progress = ToolProgressCadence {
+ after_ms: 25,
+ every_ms: 25,
+ };
+ let execution = tokio::spawn(async move {
+ execute_prepared_tool_call(&prepared, progress, None, &sink).await;
+ });
+ let count = || *ticks.lock().unwrap();
+ // Bounded first-tick wait: a broken heartbeat fails the test
+ // instead of hanging the suite.
+ let first_tick = tokio::time::timeout(std::time::Duration::from_secs(2), async {
+ while count() == 0 {
+ tokio::time::sleep(std::time::Duration::from_millis(5)).await;
+ }
+ })
+ .await;
+ assert!(
+ first_tick.is_ok(),
+ "the heartbeat never ticked within 2s at a 25ms cadence"
+ );
+ // Cancel the frame mid-await: the Drop guard (not the explicit
+ // stop) must abort the heartbeat.
+ execution.abort();
+ // Awaiting the cancelled handle settles only once the frame (and
+ // its HeartbeatGuard) has dropped, so the abort is in effect; the
+ // settle window absorbs an in-flight tick before sampling.
+ let _ = execution.await;
+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+ let after_cancel = count();
+ tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+ assert_eq!(
+ count(),
+ after_cancel,
+ "the heartbeat must stop when the call frame is cancelled"
+ );
+ }
+}
diff --git a/crates/pa-agent/src/agent_loop/tools.rs b/crates/pa-agent/src/agent_loop/tools.rs
index 09036e188a..1d5557dce7 100644
--- a/crates/pa-agent/src/agent_loop/tools.rs
+++ b/crates/pa-agent/src/agent_loop/tools.rs
@@ -116,7 +116,8 @@ async fn execute_tool_calls_sequential(
is_error,
},
Preparation::Prepared(prepared) => {
- let executed = execute_prepared_tool_call(&prepared, signal, emit).await;
+ let executed =
+ execute_prepared_tool_call(&prepared, config.tool_progress, signal, emit).await;
finalize_executed_tool_call(
current_context,
assistant_message,
@@ -205,8 +206,13 @@ async fn execute_tool_calls_parallel(
args: prepared.args.clone(),
};
let handle = tokio::spawn(async move {
- let executed =
- execute_prepared_tool_call(&prepared, Some(&signal), &sink).await;
+ let executed = execute_prepared_tool_call(
+ &prepared,
+ config.tool_progress,
+ Some(&signal),
+ &sink,
+ )
+ .await;
let finalized = finalize_executed_tool_call(
&context,
&assistant_message,
diff --git a/crates/pa-agent/src/types.rs b/crates/pa-agent/src/types.rs
index f8dada2b63..f05acc84b7 100644
--- a/crates/pa-agent/src/types.rs
+++ b/crates/pa-agent/src/types.rs
@@ -483,6 +483,24 @@ impl AgentToolResult {
/// Callback used by tools to publish partial execution updates.
pub type AgentToolUpdateCallback = Arc<dyn Fn(AgentToolResult) + Send + Sync>;
+/// Heartbeat cadence for in-flight tool calls: quiet until `after_ms`
+/// elapses, then one [`AgentEvent::ToolExecutionProgress`] every
+/// `every_ms` while the call still runs.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ToolProgressCadence {
+ pub after_ms: u64,
+ pub every_ms: u64,
+}
+
+impl Default for ToolProgressCadence {
+ fn default() -> Self {
+ ToolProgressCadence {
+ after_ms: 60_000,
+ every_ms: 60_000,
+ }
+ }
+}
+
/// Tool definition used by the agent runtime.
///
/// The schema is a plain JSON Schema `Value` (TS validates against a `TypeBox`
@@ -559,6 +577,14 @@ pub enum AgentEvent {
tool_name: String,
args: serde_json::Value,
},
+ /// In-flight tool-call heartbeat: emitted on the
+ /// [`AgentLoopConfig::tool_progress`] cadence once a call outlives the
+ /// initial threshold, so a stalled call is visible while it runs.
+ ToolExecutionProgress {
+ tool_call_id: String,
+ tool_name: String,
+ elapsed_ms: u64,
+ },
ToolExecutionUpdate {
tool_call_id: String,
tool_name: String,
diff --git a/crates/pa-agent/tests/agent_loop_scripted.rs b/crates/pa-agent/tests/agent_loop_scripted.rs
index f9a1814236..36ef87cf52 100644
--- a/crates/pa-agent/tests/agent_loop_scripted.rs
+++ b/crates/pa-agent/tests/agent_loop_scripted.rs
@@ -173,6 +173,7 @@ fn event_type(event: &AgentEvent) -> &'static str {
AgentEvent::MessageUpdate { .. } => "message_update",
AgentEvent::MessageEnd { .. } => "message_end",
AgentEvent::ToolExecutionStart { .. } => "tool_execution_start",
+ AgentEvent::ToolExecutionProgress { .. } => "tool_execution_progress",
AgentEvent::ToolExecutionUpdate { .. } => "tool_execution_update",
AgentEvent::ToolExecutionEnd { .. } => "tool_execution_end",
}
diff --git a/crates/pa-core/src/session/manager/append.rs b/crates/pa-core/src/session/manager/append.rs
index bb2a9324a2..73889e0374 100644
--- a/crates/pa-core/src/session/manager/append.rs
+++ b/crates/pa-core/src/session/manager/append.rs
@@ -162,6 +162,50 @@ impl SessionManager {
Ok(id)
}
+ /// # Errors
+ ///
+ /// Returns the underlying I/O error when the durable append fails.
+ pub fn append_tool_started(
+ &mut self,
+ tool_call_id: &str,
+ tool_name: &str,
+ args: serde_json::Value,
+ ) -> std::io::Result<String> {
+ let base = self.next_base();
+ let id = base.id.clone().unwrap_or_default();
+ self.append_entry(FileEntry::ToolStarted {
+ payload: pa_types::session::ToolStartedEntry {
+ tool_call_id: tool_call_id.to_string(),
+ tool_name: tool_name.to_string(),
+ args,
+ },
+ base,
+ })?;
+ Ok(id)
+ }
+
+ /// # Errors
+ ///
+ /// Returns the underlying I/O error when the durable append fails.
+ pub fn append_tool_progress(
+ &mut self,
+ tool_call_id: &str,
+ tool_name: &str,
+ elapsed_ms: u64,
+ ) -> std::io::Result<String> {
+ let base = self.next_base();
+ let id = base.id.clone().unwrap_or_default();
+ self.append_entry(FileEntry::ToolProgress {
+ payload: pa_types::session::ToolProgressEntry {
+ tool_call_id: tool_call_id.to_string(),
+ tool_name: tool_name.to_string(),
+ elapsed_ms,
+ },
+ base,
+ })?;
+ Ok(id)
+ }
+
/// # Errors
///
/// Returns the underlying I/O error when the durable append fails.
diff --git a/crates/pa-core/src/session/manager_ext.rs b/crates/pa-core/src/session/manager_ext.rs
index 273fa849bf..dcfbd002bc 100644
--- a/crates/pa-core/src/session/manager_ext.rs
+++ b/crates/pa-core/src/session/manager_ext.rs
@@ -269,6 +269,8 @@ fn entry_type(entry: &FileEntry) -> &'static str {
match entry {
FileEntry::Header { .. } => "session",
FileEntry::Message { .. } => "message",
+ FileEntry::ToolStarted { .. } => "tool_started",
+ FileEntry::ToolProgress { .. } => "tool_progress",
FileEntry::ThinkingLevelChange { .. } => "thinking_level_change",
FileEntry::ServiceTierChange { .. } => "service_tier_change",
FileEntry::ModelChange { .. } => "model_change",
diff --git a/crates/pa-core/src/session/mod.rs b/crates/pa-core/src/session/mod.rs
index 87ff23da12..454e3f691e 100644
--- a/crates/pa-core/src/session/mod.rs
+++ b/crates/pa-core/src/session/mod.rs
@@ -225,6 +225,8 @@ fn entry_base_mut(entry: &mut FileEntry) -> Option<&mut EntryBase> {
match entry {
FileEntry::Header { .. } | FileEntry::Unknown { .. } => None,
FileEntry::Message { base, .. }
+ | FileEntry::ToolStarted { base, .. }
+ | FileEntry::ToolProgress { base, .. }
| FileEntry::ThinkingLevelChange { base, .. }
| FileEntry::ServiceTierChange { base, .. }
| FileEntry::ModelChange { base, .. }
diff --git a/crates/pa-core/src/session_engine/mod.rs b/crates/pa-core/src/session_engine/mod.rs
index 8e7485f554..bf78c62840 100644
--- a/crates/pa-core/src/session_engine/mod.rs
+++ b/crates/pa-core/src/session_engine/mod.rs
@@ -407,6 +407,36 @@ async fn persist_event(
eprintln!("pa-core: message row not persisted: {error}");
}
}
+ // Tool-call lifecycle rows: the start and running sides of the
+ // trace (the completion side is the toolResult message row). Like
+ // the message arm, a failed durable write never fails the run.
+ AgentEvent::ToolExecutionStart {
+ tool_call_id,
+ tool_name,
+ args,
+ } => {
+ let result = session
+ .lock()
+ .await
+ .append_tool_started(&tool_call_id, &tool_name, args);
+ if let Err(error) = result {
+ eprintln!("pa-core: tool_started row not persisted: {error}");
+ }
+ }
+ AgentEvent::ToolExecutionProgress {
+ tool_call_id,
+ tool_name,
+ elapsed_ms,
+ } => {
+ let result =
+ session
+ .lock()
+ .await
+ .append_tool_progress(&tool_call_id, &tool_name, elapsed_ms);
+ if let Err(error) = result {
+ eprintln!("pa-core: tool_progress row not persisted: {error}");
+ }
+ }
// Git state is captured at both run boundaries, exactly like the
// TS run-boundary event path: a commit or branch switch made during the run
// (e.g. via the bash tool) lands in the session file at `agent_end`.
diff --git a/crates/pa-core/src/session_engine/session_events.rs b/crates/pa-core/src/session_engine/session_events.rs
index 22a188b01a..ec34633d1f 100644
--- a/crates/pa-core/src/session_engine/session_events.rs
+++ b/crates/pa-core/src/session_engine/session_events.rs
@@ -166,6 +166,16 @@ pub fn agent_event_json(event: &AgentEvent) -> Option<serde_json::Value> {
"toolName": tool_name,
"args": args,
}),
+ AgentEvent::ToolExecutionProgress {
+ tool_call_id,
+ tool_name,
+ elapsed_ms,
+ } => serde_json::json!({
+ "type": "tool_execution_progress",
+ "toolCallId": tool_call_id,
+ "toolName": tool_name,
+ "elapsedMs": elapsed_ms,
+ }),
AgentEvent::ToolExecutionUpdate {
tool_call_id,
tool_name,
diff --git a/crates/pa-core/src/session_engine/telemetry.rs b/crates/pa-core/src/session_engine/telemetry.rs
index 50b5196c9d..53cc6d150d 100644
--- a/crates/pa-core/src/session_engine/telemetry.rs
+++ b/crates/pa-core/src/session_engine/telemetry.rs
@@ -927,7 +927,9 @@ fn handle_event(
// Both are still run-scoped events: while the cached decision is
// off they sever (a tool whose execution spans an opt-out observed
// at a boundary never counts into a surviving run).
- AgentEvent::TurnEnd { .. } | AgentEvent::ToolExecutionUpdate { .. } => {}
+ AgentEvent::TurnEnd { .. }
+ | AgentEvent::ToolExecutionUpdate { .. }
+ | AgentEvent::ToolExecutionProgress { .. } => {}
}
}
diff --git a/crates/pa-core/src/session_engine/tests.rs b/crates/pa-core/src/session_engine/tests.rs
index 520fcac9f6..2d3a693a9c 100644
--- a/crates/pa-core/src/session_engine/tests.rs
+++ b/crates/pa-core/src/session_engine/tests.rs
@@ -826,6 +826,352 @@ async fn prompt_persists_tool_results() {
assert!(entry["timestamp"].as_str().is_some());
}
+/// The tool-call lifecycle trace: a `tool_started` row at call issuance
+/// (call id, tool name, the command text), `tool_progress` heartbeat ticks
+/// while a long call runs, and the completed result carrying its duration.
+/// A stalled call is visible in the session trace while it runs — the
+/// Oscar incident showed a 13-minute recursive grep with a blank trace
+/// between issuance and the user's cancel.
+#[tokio::test]
+async fn long_tool_call_persists_started_tick_and_duration_rows() {
+ struct SlowTool;
+ impl pa_agent::types::AgentTool for SlowTool {
+ fn name(&self) -> &'static str {
+ "slow"
+ }
+ fn description(&self) -> &'static str {
+ "sleeps before answering"
+ }
+ fn parameters(&self) -> &serde_json::Value {
+ static PARAMETERS: std::sync::OnceLock<serde_json::Value> = std::sync::OnceLock::new();
+ PARAMETERS.get_or_init(|| serde_json::json!({ "type": "object" }))
+ }
+ fn execute(
+ self: Arc<Self>,
+ _tool_call_id: String,
+ _params: serde_json::Value,
+ _signal: pa_agent::abort::AbortSignal,
+ _on_update: pa_agent::types::AgentToolUpdateCallback,
+ ) -> pa_agent::BoxFut<'static, anyhow::Result<pa_agent::types::AgentToolResult>> {
+ Box::pin(async {
+ let started = std::time::Instant::now();
+ // Staged long call: ~5s run with the tick threshold pinned
+ // at 1s, so the heartbeat fires well before completion.
+ tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ let mut result = pa_agent::types::AgentToolResult::text("done");
+ result.details = serde_json::json!({
+ "status": "ok",
+ "durationMs": started.elapsed().as_millis() as u64,
+ });
+ Ok(result)
+ })
+ }
+ }
+ let provider = Arc::new(ScriptedProvider::new(test_model()));
+ provider.push_tool_call_turn(
+ Some("calling"),
+ vec![(
+ "call-1",
+ "slow",
+ serde_json::json!({ "command": "grep -rli training ~/" }),
+ )],
+ );
+ provider.push_text_turn("done");
+ let options = AgentOptions {
+ initial_state: AgentInitialState {
+ model: Some(test_model()),
+ ..Default::default()
+ },
+ stream_fn: Some(provider.stream_fn()),
+ tool_progress: Some(pa_agent::types::ToolProgressCadence {
+ after_ms: 1_000,
+ every_ms: 1_000,
+ }),
+ ..Default::default()
+ };
+ let agent = Agent::new(options);
+ agent.set_tools(vec![Arc::new(SlowTool)]).await;
+ let tmp = tempfile::tempdir().unwrap();
+ let session = SessionManager::persisted(std::path::Path::new("/w"), tmp.path());
+ let engine = AgentSession::new(Arc::new(agent), session, vec![])
+ .await
+ .unwrap();
+ engine.prompt("hi", PromptOptions::default()).await.unwrap();
+ engine.agent().wait_for_idle().await;
+ let entries = engine.entries().await;
+
+ let started = entries
+ .iter()
+ .find_map(|entry| match entry {
+ FileEntry::ToolStarted { payload, .. } => Some(payload.clone()),
+ _ => None,
+ })
+ .expect("tool_started row persisted");
+ assert_eq!(started.tool_call_id, "call-1");
+ assert_eq!(started.tool_name, "slow");
+ assert_eq!(
+ started.args,
+ serde_json::json!({ "command": "grep -rli training ~/" })
+ );
+
+ let ticks: Vec<pa_types::session::ToolProgressEntry> = entries
+ .iter()
+ .filter_map(|entry| match entry {
+ FileEntry::ToolProgress { payload, .. } => Some(payload.clone()),
+ _ => None,
+ })
+ .collect();
+ assert!(
+ !ticks.is_empty(),
+ "a 5s call with a 1s tick must emit at least one tool_progress row"
+ );
+ assert!(ticks.len() <= 5, "ticks bounded by the cadence: {ticks:?}");
+ assert!(ticks.iter().all(|tick| tick.tool_call_id == "call-1"));
+ assert!(ticks.iter().all(|tick| tick.tool_name == "slow"));
+ assert!(
+ ticks
+ .iter()
+ .all(|tick| tick.elapsed_ms >= 900 && tick.elapsed_ms < 5_500),
+ "tick elapsed is wall-clock time since the call started: {ticks:?}"
+ );
+
+ let tool_result = entries
+ .iter()
+ .find_map(|entry| match entry {
+ FileEntry::Message {
+ message: SessionAgentMessage::ToolResult(result),
+ ..
+ } => Some(result.clone()),
+ _ => None,
+ })
+ .expect("toolResult entry persisted");
+ assert!(!tool_result.is_error);
+ assert_eq!(
+ tool_result.details.as_ref().and_then(|d| d.get("status")),
+ Some(&serde_json::json!("ok"))
+ );
+ let duration = tool_result
+ .details
+ .as_ref()
+ .and_then(|d| d.get("durationMs"))
+ .and_then(serde_json::Value::as_u64)
+ .expect("completed result carries durationMs");
+ assert!(duration >= 4_500, "durationMs: {duration}");
+
+ // The rows also live on disk in the session jsonl (the trace the
+ // session zip contains), with the new types round-tripping.
+ let file = tmp
+ .path()
+ .join(format!("{}.jsonl", engine.session_id().await))
+ .to_string_lossy()
+ .to_string();
+ let lines: Vec<serde_json::Value> = std::fs::read_to_string(file)
+ .unwrap()
+ .lines()
+ .map(|line| serde_json::from_str(line).unwrap())
+ .collect();
+ let started_line = lines
+ .iter()
+ .find(|line| line.get("type") == Some(&serde_json::json!("tool_started")))
+ .expect("tool_started line on disk");
+ assert_eq!(started_line["toolCallId"], "call-1");
+ assert_eq!(started_line["toolName"], "slow");
+ assert_eq!(started_line["args"]["command"], "grep -rli training ~/");
+ assert!(
+ lines
+ .iter()
+ .any(|line| line.get("type") == Some(&serde_json::json!("tool_progress"))),
+ "tool_progress line on disk"
+ );
+}
+
+/// The short-call contract: the start row and the result row persist, and
+/// the heartbeat stays quiet (no `tool_progress` row under the threshold).
+#[tokio::test]
+async fn short_tool_call_persists_started_row_without_progress() {
+ struct FastTool;
+ impl pa_agent::types::AgentTool for FastTool {
+ fn name(&self) -> &'static str {
+ "fast"
+ }
+ fn description(&self) -> &'static str {
+ "answers immediately"
+ }
+ fn parameters(&self) -> &serde_json::Value {
+ static PARAMETERS: std::sync::OnceLock<serde_json::Value> = std::sync::OnceLock::new();
+ PARAMETERS.get_or_init(|| serde_json::json!({ "type": "object" }))
+ }
+ fn execute(
+ self: Arc<Self>,
+ _tool_call_id: String,
+ _params: serde_json::Value,
+ _signal: pa_agent::abort::AbortSignal,
+ _on_update: pa_agent::types::AgentToolUpdateCallback,
+ ) -> pa_agent::BoxFut<'static, anyhow::Result<pa_agent::types::AgentToolResult>> {
+ Box::pin(async { Ok(pa_agent::types::AgentToolResult::text("done")) })
+ }
+ }
+ let provider = Arc::new(ScriptedProvider::new(test_model()));
+ provider.push_tool_call_turn(
+ Some("calling"),
+ vec![("call-1", "fast", serde_json::json!({}))],
+ );
+ provider.push_text_turn("done");
+ let options = AgentOptions {
+ initial_state: AgentInitialState {
+ model: Some(test_model()),
+ ..Default::default()
+ },
+ stream_fn: Some(provider.stream_fn()),
+ tool_progress: Some(pa_agent::types::ToolProgressCadence {
+ after_ms: 1_000,
+ every_ms: 1_000,
+ }),
+ ..Default::default()
+ };
+ let agent = Agent::new(options);
+ agent.set_tools(vec![Arc::new(FastTool)]).await;
+ let tmp = tempfile::tempdir().unwrap();
+ let session = SessionManager::in_memory(tmp.path());
+ let engine = AgentSession::new(Arc::new(agent), session, vec![])
+ .await
+ .unwrap();
+ engine.prompt("hi", PromptOptions::default()).await.unwrap();
+ engine.agent().wait_for_idle().await;
+ let entries = engine.entries().await;
+ assert!(
+ entries.iter().any(|entry| matches!(entry, FileEntry::ToolStarted { payload, .. } if payload.tool_call_id == "call-1")),
+ "short call still persists its tool_started row"
+ );
+ assert!(
+ entries
+ .iter()
+ .all(|entry| !matches!(entry, FileEntry::ToolProgress { .. })),
+ "a fast call must not emit tool_progress rows"
+ );
+ assert!(
+ entries.iter().any(|entry| matches!(
+ entry,
+ FileEntry::Message {
+ message: SessionAgentMessage::ToolResult(_),
+ ..
+ }
+ )),
+ "short call persists its toolResult row"
+ );
+}
+
+/// The aborted completion carries the elapsed duration: the trace shows how
+/// long the call ran before the kill, not just that it died. Before this
+/// fix, the aborted toolResult row carried only the bare "Tool execution
+/// aborted" text with no duration.
+#[tokio::test]
+async fn aborted_tool_call_result_carries_elapsed_duration() {
+ struct StalledTool(Arc<std::sync::atomic::AtomicBool>);
+ impl pa_agent::types::AgentTool for StalledTool {
+ fn name(&self) -> &'static str {
+ "stalled"
+ }
+ fn description(&self) -> &'static str {
+ "never finishes on its own"
+ }
+ fn parameters(&self) -> &serde_json::Value {
+ static PARAMETERS: std::sync::OnceLock<serde_json::Value> = std::sync::OnceLock::new();
+ PARAMETERS.get_or_init(|| serde_json::json!({ "type": "object" }))
+ }
+ fn execute(
+ self: Arc<Self>,
+ _tool_call_id: String,
+ _params: serde_json::Value,
+ _signal: pa_agent::abort::AbortSignal,
+ _on_update: pa_agent::types::AgentToolUpdateCallback,
+ ) -> pa_agent::BoxFut<'static, anyhow::Result<pa_agent::types::AgentToolResult>> {
+ let entered = Arc::clone(&self.0);
+ Box::pin(async move {
+ entered.store(true, std::sync::atomic::Ordering::SeqCst);
+ // The recursive-scan shape: unbounded runtime, only the
+ // abort ends it.
+ tokio::time::sleep(std::time::Duration::from_secs(600)).await;
+ Ok(pa_agent::types::AgentToolResult::text("done"))
+ })
+ }
+ }
+ let entered = Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let provider = Arc::new(ScriptedProvider::new(test_model()));
+ provider.push_tool_call_turn(
+ Some("calling"),
+ vec![("call-1", "stalled", serde_json::json!({}))],
+ );
+ provider.push_text_turn("done");
+ let options = AgentOptions {
+ initial_state: AgentInitialState {
+ model: Some(test_model()),
+ ..Default::default()
+ },
+ stream_fn: Some(provider.stream_fn()),
+ ..Default::default()
+ };
+ let agent = Agent::new(options);
+ agent
+ .set_tools(vec![Arc::new(StalledTool(Arc::clone(&entered)))])
+ .await;
+ let tmp = tempfile::tempdir().unwrap();
+ let session = SessionManager::in_memory(tmp.path());
+ let engine = Arc::new(
+ AgentSession::new(Arc::new(agent), session, vec![])
+ .await
+ .unwrap(),
+ );
+ let prompt_engine = Arc::clone(&engine);
+ let prompting =
+ tokio::spawn(async move { prompt_engine.prompt("hi", PromptOptions::default()).await });
+ // Bounded entered wait: a stalled tool that never starts fails the
+ // test instead of hanging the suite.
+ let tool_entered = tokio::time::timeout(std::time::Duration::from_secs(2), async {
+ while !entered.load(std::sync::atomic::Ordering::SeqCst) {
+ tokio::time::sleep(std::time::Duration::from_millis(10)).await;
+ }
+ })
+ .await;
+ assert!(
+ tool_entered.is_ok(),
+ "the stalled tool never entered within 2s"
+ );
+ // Let the call run briefly so the elapsed duration is non-zero.
+ tokio::time::sleep(std::time::Duration::from_millis(150)).await;
+ engine.agent().abort();
+ prompting.await.unwrap().unwrap();
+ engine.agent().wait_for_idle().await;
+ let entries = engine.entries().await;
+ let tool_result = entries
+ .iter()
+ .find_map(|entry| match entry {
+ FileEntry::Message {
+ message: SessionAgentMessage::ToolResult(result),
+ ..
+ } => Some(result.clone()),
+ _ => None,
+ })
+ .expect("aborted toolResult entry persisted");
+ assert!(tool_result.is_error);
+ match &tool_result.content[..] {
+ [pa_types::ai::UserContentBlock::Text(text)] => {
+ assert_eq!(text.text, "Tool execution aborted");
+ }
+ other => panic!("expected the abort error text: {other:?}"),
+ }
+ let details = tool_result
+ .details
+ .as_ref()
+ .expect("aborted result has details");
+ assert_eq!(details.get("status"), Some(&serde_json::json!("aborted")));
+ let duration = details
+ .get("durationMs")
+ .and_then(serde_json::Value::as_u64)
+ .expect("aborted result carries durationMs");
+ assert!(duration >= 100, "durationMs: {duration}");
+}
+
/// Persisted `ipython`/`python` calls dispatch through `ToolDefinitionBridge`
/// to the registered Python tool.
#[tokio::test]
diff --git a/crates/pa-daemon/src/agent_engine/turn/run_once.rs b/crates/pa-daemon/src/agent_engine/turn/run_once.rs
index 95b7354780..7777bb36cb 100644
--- a/crates/pa-daemon/src/agent_engine/turn/run_once.rs
+++ b/crates/pa-daemon/src/agent_engine/turn/run_once.rs
@@ -295,6 +295,7 @@ impl AgentSessionEngine {
args: args.clone(),
});
}
+ AgentEvent::ToolExecutionProgress { .. } => {}
AgentEvent::ToolExecutionUpdate {
tool_call_id,
partial_result,
diff --git a/crates/pa-tui/src/tree_display.rs b/crates/pa-tui/src/tree_display.rs
index 9c9588f996..d1627e4b99 100644
--- a/crates/pa-tui/src/tree_display.rs
+++ b/crates/pa-tui/src/tree_display.rs
@@ -297,6 +297,18 @@ pub fn entry_display_text<S: std::hash::BuildHasher + Default>(
color(ThemeColor::Dim, "]".to_string()),
]
}
+ FileEntry::ToolStarted { payload, .. } => vec![color(
+ ThemeColor::Dim,
+ format!("[tool started: {}]", payload.tool_name),
+ )],
+ FileEntry::ToolProgress { payload, .. } => vec![color(
+ ThemeColor::Dim,
+ format!(
+ "[tool running: {} ({}s)]",
+ payload.tool_name,
+ payload.elapsed_ms / 1000
+ ),
+ )],
FileEntry::Header { .. }
| FileEntry::SessionState { .. }
| FileEntry::GitState { .. }
diff --git a/crates/pa-types/src/session.rs b/crates/pa-types/src/session.rs
index a9f0bfbb68..0f266c36b7 100644
--- a/crates/pa-types/src/session.rs
+++ b/crates/pa-types/src/session.rs
@@ -300,6 +300,27 @@ pub struct GitStateEntry {
pub git: GitContext,
}
+/// `type: "tool_started"`: a tool call issued to a tool (call id, tool name,
+/// the call arguments) written when the call starts, so the trace shows an
+/// in-flight call before its result row exists.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ToolStartedEntry {
+ pub tool_call_id: String,
+ pub tool_name: String,
+ pub args: Value,
+}
+
+/// `type: "tool_progress"`: heartbeat row for a still-running tool call
+/// (`elapsedMs` since the call started); quiet for short calls.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ToolProgressEntry {
+ pub tool_call_id: String,
+ pub tool_name: String,
+ pub elapsed_ms: u64,
+}
+
/// `type: "custom_message"`: a custom message preserved verbatim for replay
/// (content/details/display round-trip through session restore).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -337,6 +358,18 @@ pub enum FileEntry {
#[serde(flatten)]
base: EntryBase,
},
+ ToolStarted {
+ #[serde(flatten)]
+ payload: ToolStartedEntry,
+ #[serde(flatten)]
+ base: EntryBase,
+ },
+ ToolProgress {
+ #[serde(flatten)]
+ payload: ToolProgressEntry,
+ #[serde(flatten)]
+ base: EntryBase,
+ },
ThinkingLevelChange {
#[serde(flatten)]
payload: ThinkingLevelChangeEntry,
@@ -436,6 +469,18 @@ enum KnownFileEntry {
#[serde(flatten)]
base: EntryBase,
},
+ ToolStarted {
+ #[serde(flatten)]
+ payload: ToolStartedEntry,
+ #[serde(flatten)]
+ base: EntryBase,
+ },
+ ToolProgress {
+ #[serde(flatten)]
+ payload: ToolProgressEntry,
+ #[serde(flatten)]
+ base: EntryBase,
+ },
ThinkingLevelChange {
#[serde(flatten)]
payload: ThinkingLevelChangeEntry,
@@ -515,6 +560,8 @@ impl From<KnownFileEntry> for FileEntry {
match entry {
KnownFileEntry::Header { header } => Self::Header { header },
KnownFileEntry::Message { message, base } => Self::Message { message, base },
+ KnownFileEntry::ToolStarted { payload, base } => Self::ToolStarted { payload, base },
+ KnownFileEntry::ToolProgress { payload, base } => Self::ToolProgress { payload, base },
KnownFileEntry::ThinkingLevelChange { payload, base } => {
Self::ThinkingLevelChange { payload, base }
}
@@ -565,6 +612,8 @@ impl FileEntry {
return rest.get("id").and_then(|v| v.as_str());
}
FileEntry::Message { base, .. }
+ | FileEntry::ToolStarted { base, .. }
+ | FileEntry::ToolProgress { base, .. }
| FileEntry::ThinkingLevelChange { base, .. }
| FileEntry::ServiceTierChange { base, .. }
| FileEntry::ModelChange { base, .. }
@@ -590,6 +639,8 @@ impl FileEntry {
return rest.get("parentId").and_then(|v| v.as_str());
}
FileEntry::Message { base, .. }
+ | FileEntry::ToolStarted { base, .. }
+ | FileEntry::ToolProgress { base, .. }
| FileEntry::ThinkingLevelChange { base, .. }
| FileEntry::ServiceTierChange { base, .. }
| FileEntry::ModelChange { base, .. }
@@ -618,6 +669,8 @@ impl FileEntry {
.unwrap_or_default();
}
FileEntry::Message { base, .. }
+ | FileEntry::ToolStarted { base, .. }
+ | FileEntry::ToolProgress { base, .. }
| FileEntry::ThinkingLevelChange { base, .. }
| FileEntry::ServiceTierChange { base, .. }
| FileEntry::ModelChange { base, .. }