A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
[RSI, feature] trace: tool-call lifecycle events in the session trace (started, heartbeat, elapsed-on-abort) (#3689)
Seth Kartenda4ab9978b97
diff --git a/crates/pa-agent/src/agent.rs b/crates/pa-agent/src/agent.rsindex d60b861cfd..eb859b3f04 100644--- a/crates/pa-agent/src/agent.rs+++ b/crates/pa-agent/src/agent.rs@@ -17,7 +17,7 @@ use crate::agent_loop::{use crate::stream::StreamFn;use crate::types::{AgentEvent, AgentMessage, AgentTool, ImageContent, Model, ThinkingLevel, ToolExecutionMode,- Usage,+ ToolProgressCadence, Usage,};#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]@@ -102,6 +102,9 @@ pub struct AgentOptions {pub follow_up_mode: Option<QueueMode>,pub session_id: Option<String>,pub tool_execution: Option<ToolExecutionMode>,+ /// Heartbeat cadence for in-flight tool calls; `None` keeps the 60s+ /// default tick.+ pub tool_progress: Option<ToolProgressCadence>,}struct MutableAgentState {@@ -345,6 +348,7 @@ struct AgentInner {model_override: Mutex<Option<AgentModelOverride>>,session_id: Option<String>,tool_execution: ToolExecutionMode,+ tool_progress: ToolProgressCadence,}impl AgentInner {@@ -406,7 +410,8 @@ impl AgentInner {}AgentEvent::AgentStart| AgentEvent::TurnStart- | AgentEvent::ToolExecutionUpdate { .. } => {}+ | AgentEvent::ToolExecutionUpdate { .. }+ | AgentEvent::ToolExecutionProgress { .. } => {}}let Some(signal) = self.current_signal() else {@@ -568,6 +573,7 @@ impl AgentInner {config.get_follow_up_messages = Some(follow_up);config.get_continuation_messages = continuation;config.tool_execution = self.tool_execution;+ config.tool_progress = self.tool_progress;config.before_tool_call.clone_from(&self.before_tool_call);config.after_tool_call.clone_from(&self.after_tool_call);config@@ -797,6 +803,7 @@ impl Agent {tool_execution: options.tool_execution.unwrap_or(ToolExecutionMode::Parallel),+ tool_progress: options.tool_progress.unwrap_or_default(),});Agent { inner }}diff --git a/crates/pa-agent/src/agent_loop/mod.rs b/crates/pa-agent/src/agent_loop/mod.rsindex 86cc0985e7..a503a73eb6 100644--- a/crates/pa-agent/src/agent_loop/mod.rs+++ b/crates/pa-agent/src/agent_loop/mod.rs@@ -109,6 +109,9 @@ pub struct AgentLoopConfig {pub get_follow_up_messages: Option<PollMessagesFn>,pub get_continuation_messages: Option<GetContinuationMessagesFn>,pub tool_execution: ToolExecutionMode,+ /// Heartbeat cadence for in-flight tool calls (start-side trace+ /// visibility for long calls); defaults to a 60s tick.+ pub tool_progress: crate::types::ToolProgressCadence,pub before_tool_call: Option<BeforeToolCallFn>,pub after_tool_call: Option<AfterToolCallFn>,}@@ -135,6 +138,7 @@ impl AgentLoopConfig {get_follow_up_messages: None,get_continuation_messages: None,tool_execution: ToolExecutionMode::Parallel,+ tool_progress: crate::types::ToolProgressCadence::default(),before_tool_call: None,after_tool_call: None,}diff --git a/crates/pa-agent/src/agent_loop/tool_call.rs b/crates/pa-agent/src/agent_loop/tool_call.rsindex 13ec058a29..5ce54ae75a 100644--- a/crates/pa-agent/src/agent_loop/tool_call.rs+++ b/crates/pa-agent/src/agent_loop/tool_call.rs@@ -7,7 +7,8 @@ use std::sync::Arc;use crate::abort::{is_abort_error, AbortSignal};use crate::types::{AfterToolCallContext, AgentContext, AgentEvent, AgentMessage, AgentTool, AgentToolResult,- AgentToolUpdateCallback, AssistantMessage, BeforeToolCallContext, ToolCall, ToolResultMessage,+ AgentToolUpdateCallback, AssistantMessage, BeforeToolCallContext, ToolCall,+ ToolProgressCadence, ToolResultMessage,};use super::abort::race_with_abort;@@ -114,6 +115,7 @@ pub(crate) struct ExecutedToolCallOutcome {/// the tool fails (or aborts, with the TS message "Tool execution aborted").pub(crate) async fn execute_prepared_tool_call(prepared: &PreparedToolCall,+ progress: ToolProgressCadence,signal: Option<&AbortSignal>,emit: &AgentEventSink,) -> ExecutedToolCallOutcome {@@ -125,6 +127,19 @@ pub(crate) async fn execute_prepared_tool_call(let tool_name = prepared.tool.name().to_string();let args = prepared.tool_call.arguments.clone();let sink = Arc::clone(emit);+ let started_at = std::time::Instant::now();+ // Lifecycle heartbeat: quiet until `progress.after_ms`, then one+ // `ToolExecutionProgress` event every `progress.every_ms` while the call+ // still runs. The guard aborts the task on EVERY exit path — return,+ // panic, and cancellation of this frame — so the unbounded tick loop+ // never leaks as a detached task.+ let mut heartbeat = HeartbeatGuard(Some(spawn_tool_heartbeat(+ Arc::clone(emit),+ tool_call_id.clone(),+ tool_name.clone(),+ started_at,+ progress,+ )));let signal_for_updates = signal.cloned().unwrap_or_default();let emit_updates = tokio::spawn(async move {while let Some(partial_result) = update_rx.recv().await {@@ -161,41 +176,22 @@ pub(crate) async fn execute_prepared_tool_call(signal,).await;+ heartbeat.stop();+ let elapsed_ms = elapsed_ms_since(started_at);accepting_updates.store(false, std::sync::atomic::Ordering::SeqCst);drop(update_tx);match execute_result {Ok(result) => {- match race_with_abort(- async {- emit_updates.await.map_err(|error| {- anyhow::anyhow!("Tool update emitter task failed: {error}")- })- },- signal,- )- .await- {- Ok(update_result) => {- if let Err(error) = update_result {- // Success path: a failing update emitter mirrors a- // rejecting updateEvents promise in TS.- return ExecutedToolCallOutcome {- result: error_tool_result(signal, &error),- is_error: true,- };- }- }- Err(error) => {- let aborted = signal.is_some_and(AbortSignal::is_aborted);- if !(aborted && is_abort_error(&error)) {- return ExecutedToolCallOutcome {- result: error_tool_result(signal, &error),- is_error: true,- };- }- }+ // Success path: a failing update emitter mirrors a rejecting+ // updateEvents promise in TS; an aborted drain (the call itself+ // succeeded) settles cleanly.+ if let Err(error) = settle_success_drain(emit_updates, signal).await {+ return ExecutedToolCallOutcome {+ result: error_tool_result(signal, &error, elapsed_ms),+ is_error: true,+ };}ExecutedToolCallOutcome {result,@@ -216,19 +212,113 @@ pub(crate) async fn execute_prepared_tool_call().await;ExecutedToolCallOutcome {- result: error_tool_result(signal, &error),+ result: error_tool_result(signal, &error, elapsed_ms),is_error: true,}}}}-fn error_tool_result(signal: Option<&AbortSignal>, error: &anyhow::Error) -> AgentToolResult {- if signal.is_some_and(AbortSignal::is_aborted) {+/// Aborts the heartbeat task on every exit path. Dropping the bare+/// `JoinHandle` only detaches the unbounded tick task: on a panic or a+/// cancellation (a dropped `execute_prepared_tool_call` future) the explicit+/// `stop()` never runs, and the detached task would keep emitting+/// `ToolExecutionProgress` — and pin the event sink — forever.+struct HeartbeatGuard(Option<tokio::task::JoinHandle<()>>);++impl HeartbeatGuard {+ /// Stop the heartbeat on the normal path (ticks stop before the+ /// drain/result emission); `drop` covers panic and cancellation.+ fn stop(&mut self) {+ if let Some(handle) = self.0.take() {+ handle.abort();+ }+ }+}++impl Drop for HeartbeatGuard {+ fn drop(&mut self) {+ self.stop();+ }+}++/// Drain the update emitter on the success path, racing the drain against+/// abort: an aborted drain settles cleanly, any other failure surfaces as+/// the call's error.+async fn settle_success_drain(+ emit_updates: tokio::task::JoinHandle<anyhow::Result<()>>,+ signal: Option<&AbortSignal>,+) -> anyhow::Result<()> {+ match race_with_abort(+ async {+ emit_updates+ .await+ .map_err(|error| anyhow::anyhow!("Tool update emitter task failed: {error}"))+ },+ signal,+ )+ .await+ {+ Ok(update_result) => update_result,+ Err(error) => {+ let aborted = signal.is_some_and(AbortSignal::is_aborted);+ if aborted && is_abort_error(&error) {+ Ok(())+ } else {+ Err(error)+ }+ }+ }+}++/// Elapsed milliseconds since a call started (millis fit `u64`; the+/// saturating conversion only guards the type-level truncation lint).+fn elapsed_ms_since(started_at: std::time::Instant) -> u64 {+ u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX)+}++/// The heartbeat task behind the `ToolExecutionProgress` events: quiet until+/// `progress.after_ms`, then one event every `progress.every_ms` while the+/// call runs; aborted by the caller once the execution race settles.+fn spawn_tool_heartbeat(+ sink: AgentEventSink,+ tool_call_id: String,+ tool_name: String,+ started_at: std::time::Instant,+ progress: ToolProgressCadence,+) -> tokio::task::JoinHandle<()> {+ tokio::spawn(async move {+ tokio::time::sleep(std::time::Duration::from_millis(progress.after_ms)).await;+ loop {+ let _ = sink(AgentEvent::ToolExecutionProgress {+ tool_call_id: tool_call_id.clone(),+ tool_name: tool_name.clone(),+ elapsed_ms: elapsed_ms_since(started_at),+ })+ .await;+ tokio::time::sleep(std::time::Duration::from_millis(progress.every_ms)).await;+ }+ })+}++fn error_tool_result(+ signal: Option<&AbortSignal>,+ error: &anyhow::Error,+ elapsed_ms: u64,+) -> AgentToolResult {+ let aborted = signal.is_some_and(AbortSignal::is_aborted);+ let mut result = if aborted {AgentToolResult::error("Tool execution aborted")} else {AgentToolResult::error(format!("{error:#}"))- }+ };+ // The aborted/killed completion carries the elapsed duration: without+ // it the trace shows an aborted call but never how long it ran.+ result.details = serde_json::json!({+ "status": if aborted { "aborted" } else { "error" },+ "durationMs": elapsed_ms,+ });+ result}/// Apply the `afterToolCall` overrides.@@ -335,3 +425,98 @@ pub(crate) async fn emit_tool_result_message(}).await}+#[cfg(test)]+mod tests {+ use super::*;+ use std::sync::Mutex;++ struct NeverTool;++ impl AgentTool for NeverTool {+ fn name(&self) -> &'static str {+ "never"+ }+ fn description(&self) -> &'static str {+ "never finishes on its own"+ }+ fn parameters(&self) -> &serde_json::Value {+ static PARAMETERS: std::sync::OnceLock<serde_json::Value> = std::sync::OnceLock::new();+ PARAMETERS.get_or_init(|| serde_json::json!({ "type": "object" }))+ }+ fn execute(+ self: Arc<Self>,+ _tool_call_id: String,+ _params: serde_json::Value,+ _signal: crate::abort::AbortSignal,+ _on_update: AgentToolUpdateCallback,+ ) -> crate::BoxFut<'static, anyhow::Result<AgentToolResult>> {+ Box::pin(std::future::pending())+ }+ }++ /// Cancelling the call frame (a dropped `execute_prepared_tool_call`+ /// future) must stop the heartbeat task: a detached tick loop would+ /// keep emitting `ToolExecutionProgress` — and pin the event sink —+ /// after the run that owned it is gone.+ #[tokio::test]+ async fn cancelling_the_call_frame_stops_the_heartbeat() {+ let ticks: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));+ let sink: AgentEventSink = {+ let ticks = Arc::clone(&ticks);+ Arc::new(move |event| {+ let ticks = Arc::clone(&ticks);+ Box::pin(async move {+ if matches!(event, AgentEvent::ToolExecutionProgress { .. }) {+ *ticks.lock().unwrap() += 1;+ }+ Ok(())+ })+ })+ };+ let prepared = PreparedToolCall {+ tool_call: ToolCall {+ id: "call-1".to_string(),+ name: "never".to_string(),+ arguments: serde_json::json!({}),+ thought_signature: None,+ },+ tool: Arc::new(NeverTool),+ args: serde_json::json!({}),+ };+ let progress = ToolProgressCadence {+ after_ms: 25,+ every_ms: 25,+ };+ let execution = tokio::spawn(async move {+ execute_prepared_tool_call(&prepared, progress, None, &sink).await;+ });+ let count = || *ticks.lock().unwrap();+ // Bounded first-tick wait: a broken heartbeat fails the test+ // instead of hanging the suite.+ let first_tick = tokio::time::timeout(std::time::Duration::from_secs(2), async {+ while count() == 0 {+ tokio::time::sleep(std::time::Duration::from_millis(5)).await;+ }+ })+ .await;+ assert!(+ first_tick.is_ok(),+ "the heartbeat never ticked within 2s at a 25ms cadence"+ );+ // Cancel the frame mid-await: the Drop guard (not the explicit+ // stop) must abort the heartbeat.+ execution.abort();+ // Awaiting the cancelled handle settles only once the frame (and+ // its HeartbeatGuard) has dropped, so the abort is in effect; the+ // settle window absorbs an in-flight tick before sampling.+ let _ = execution.await;+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;+ let after_cancel = count();+ tokio::time::sleep(std::time::Duration::from_millis(200)).await;+ assert_eq!(+ count(),+ after_cancel,+ "the heartbeat must stop when the call frame is cancelled"+ );+ }+}diff --git a/crates/pa-agent/src/agent_loop/tools.rs b/crates/pa-agent/src/agent_loop/tools.rsindex 09036e188a..1d5557dce7 100644--- a/crates/pa-agent/src/agent_loop/tools.rs+++ b/crates/pa-agent/src/agent_loop/tools.rs@@ -116,7 +116,8 @@ async fn execute_tool_calls_sequential(is_error,},Preparation::Prepared(prepared) => {- let executed = execute_prepared_tool_call(&prepared, signal, emit).await;+ let executed =+ execute_prepared_tool_call(&prepared, config.tool_progress, signal, emit).await;finalize_executed_tool_call(current_context,assistant_message,@@ -205,8 +206,13 @@ async fn execute_tool_calls_parallel(args: prepared.args.clone(),};let handle = tokio::spawn(async move {- let executed =- execute_prepared_tool_call(&prepared, Some(&signal), &sink).await;+ let executed = execute_prepared_tool_call(+ &prepared,+ config.tool_progress,+ Some(&signal),+ &sink,+ )+ .await;let finalized = finalize_executed_tool_call(&context,&assistant_message,diff --git a/crates/pa-agent/src/types.rs b/crates/pa-agent/src/types.rsindex f8dada2b63..f05acc84b7 100644--- a/crates/pa-agent/src/types.rs+++ b/crates/pa-agent/src/types.rs@@ -483,6 +483,24 @@ impl AgentToolResult {/// Callback used by tools to publish partial execution updates.pub type AgentToolUpdateCallback = Arc<dyn Fn(AgentToolResult) + Send + Sync>;+/// Heartbeat cadence for in-flight tool calls: quiet until `after_ms`+/// elapses, then one [`AgentEvent::ToolExecutionProgress`] every+/// `every_ms` while the call still runs.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct ToolProgressCadence {+ pub after_ms: u64,+ pub every_ms: u64,+}++impl Default for ToolProgressCadence {+ fn default() -> Self {+ ToolProgressCadence {+ after_ms: 60_000,+ every_ms: 60_000,+ }+ }+}+/// Tool definition used by the agent runtime.////// The schema is a plain JSON Schema `Value` (TS validates against a `TypeBox`@@ -559,6 +577,14 @@ pub enum AgentEvent {tool_name: String,args: serde_json::Value,},+ /// In-flight tool-call heartbeat: emitted on the+ /// [`AgentLoopConfig::tool_progress`] cadence once a call outlives the+ /// initial threshold, so a stalled call is visible while it runs.+ ToolExecutionProgress {+ tool_call_id: String,+ tool_name: String,+ elapsed_ms: u64,+ },ToolExecutionUpdate {tool_call_id: String,tool_name: String,diff --git a/crates/pa-agent/tests/agent_loop_scripted.rs b/crates/pa-agent/tests/agent_loop_scripted.rsindex f9a1814236..36ef87cf52 100644--- a/crates/pa-agent/tests/agent_loop_scripted.rs+++ b/crates/pa-agent/tests/agent_loop_scripted.rs@@ -173,6 +173,7 @@ fn event_type(event: &AgentEvent) -> &'static str {AgentEvent::MessageUpdate { .. } => "message_update",AgentEvent::MessageEnd { .. } => "message_end",AgentEvent::ToolExecutionStart { .. } => "tool_execution_start",+ AgentEvent::ToolExecutionProgress { .. } => "tool_execution_progress",AgentEvent::ToolExecutionUpdate { .. } => "tool_execution_update",AgentEvent::ToolExecutionEnd { .. } => "tool_execution_end",}diff --git a/crates/pa-core/src/session/manager/append.rs b/crates/pa-core/src/session/manager/append.rsindex bb2a9324a2..73889e0374 100644--- a/crates/pa-core/src/session/manager/append.rs+++ b/crates/pa-core/src/session/manager/append.rs@@ -162,6 +162,50 @@ impl SessionManager {Ok(id)}+ /// # Errors+ ///+ /// Returns the underlying I/O error when the durable append fails.+ pub fn append_tool_started(+ &mut self,+ tool_call_id: &str,+ tool_name: &str,+ args: serde_json::Value,+ ) -> std::io::Result<String> {+ let base = self.next_base();+ let id = base.id.clone().unwrap_or_default();+ self.append_entry(FileEntry::ToolStarted {+ payload: pa_types::session::ToolStartedEntry {+ tool_call_id: tool_call_id.to_string(),+ tool_name: tool_name.to_string(),+ args,+ },+ base,+ })?;+ Ok(id)+ }++ /// # Errors+ ///+ /// Returns the underlying I/O error when the durable append fails.+ pub fn append_tool_progress(+ &mut self,+ tool_call_id: &str,+ tool_name: &str,+ elapsed_ms: u64,+ ) -> std::io::Result<String> {+ let base = self.next_base();+ let id = base.id.clone().unwrap_or_default();+ self.append_entry(FileEntry::ToolProgress {+ payload: pa_types::session::ToolProgressEntry {+ tool_call_id: tool_call_id.to_string(),+ tool_name: tool_name.to_string(),+ elapsed_ms,+ },+ base,+ })?;+ Ok(id)+ }+/// # Errors////// Returns the underlying I/O error when the durable append fails.diff --git a/crates/pa-core/src/session/manager_ext.rs b/crates/pa-core/src/session/manager_ext.rsindex 273fa849bf..dcfbd002bc 100644--- a/crates/pa-core/src/session/manager_ext.rs+++ b/crates/pa-core/src/session/manager_ext.rs@@ -269,6 +269,8 @@ fn entry_type(entry: &FileEntry) -> &'static str {match entry {FileEntry::Header { .. } => "session",FileEntry::Message { .. } => "message",+ FileEntry::ToolStarted { .. } => "tool_started",+ FileEntry::ToolProgress { .. } => "tool_progress",FileEntry::ThinkingLevelChange { .. } => "thinking_level_change",FileEntry::ServiceTierChange { .. } => "service_tier_change",FileEntry::ModelChange { .. } => "model_change",diff --git a/crates/pa-core/src/session/mod.rs b/crates/pa-core/src/session/mod.rsindex 87ff23da12..454e3f691e 100644--- a/crates/pa-core/src/session/mod.rs+++ b/crates/pa-core/src/session/mod.rs@@ -225,6 +225,8 @@ fn entry_base_mut(entry: &mut FileEntry) -> Option<&mut EntryBase> {match entry {FileEntry::Header { .. } | FileEntry::Unknown { .. } => None,FileEntry::Message { base, .. }+ | FileEntry::ToolStarted { base, .. }+ | FileEntry::ToolProgress { base, .. }| FileEntry::ThinkingLevelChange { base, .. }| FileEntry::ServiceTierChange { base, .. }| FileEntry::ModelChange { base, .. }diff --git a/crates/pa-core/src/session_engine/mod.rs b/crates/pa-core/src/session_engine/mod.rsindex 8e7485f554..bf78c62840 100644--- a/crates/pa-core/src/session_engine/mod.rs+++ b/crates/pa-core/src/session_engine/mod.rs@@ -407,6 +407,36 @@ async fn persist_event(eprintln!("pa-core: message row not persisted: {error}");}}+ // Tool-call lifecycle rows: the start and running sides of the+ // trace (the completion side is the toolResult message row). Like+ // the message arm, a failed durable write never fails the run.+ AgentEvent::ToolExecutionStart {+ tool_call_id,+ tool_name,+ args,+ } => {+ let result = session+ .lock()+ .await+ .append_tool_started(&tool_call_id, &tool_name, args);+ if let Err(error) = result {+ eprintln!("pa-core: tool_started row not persisted: {error}");+ }+ }+ AgentEvent::ToolExecutionProgress {+ tool_call_id,+ tool_name,+ elapsed_ms,+ } => {+ let result =+ session+ .lock()+ .await+ .append_tool_progress(&tool_call_id, &tool_name, elapsed_ms);+ if let Err(error) = result {+ eprintln!("pa-core: tool_progress row not persisted: {error}");+ }+ }// Git state is captured at both run boundaries, exactly like the// TS run-boundary event path: a commit or branch switch made during the run// (e.g. via the bash tool) lands in the session file at `agent_end`.diff --git a/crates/pa-core/src/session_engine/session_events.rs b/crates/pa-core/src/session_engine/session_events.rsindex 22a188b01a..ec34633d1f 100644--- a/crates/pa-core/src/session_engine/session_events.rs+++ b/crates/pa-core/src/session_engine/session_events.rs@@ -166,6 +166,16 @@ pub fn agent_event_json(event: &AgentEvent) -> Option<serde_json::Value> {"toolName": tool_name,"args": args,}),+ AgentEvent::ToolExecutionProgress {+ tool_call_id,+ tool_name,+ elapsed_ms,+ } => serde_json::json!({+ "type": "tool_execution_progress",+ "toolCallId": tool_call_id,+ "toolName": tool_name,+ "elapsedMs": elapsed_ms,+ }),AgentEvent::ToolExecutionUpdate {tool_call_id,tool_name,diff --git a/crates/pa-core/src/session_engine/telemetry.rs b/crates/pa-core/src/session_engine/telemetry.rsindex 50b5196c9d..53cc6d150d 100644--- a/crates/pa-core/src/session_engine/telemetry.rs+++ b/crates/pa-core/src/session_engine/telemetry.rs@@ -927,7 +927,9 @@ fn handle_event(// Both are still run-scoped events: while the cached decision is// off they sever (a tool whose execution spans an opt-out observed// at a boundary never counts into a surviving run).- AgentEvent::TurnEnd { .. } | AgentEvent::ToolExecutionUpdate { .. } => {}+ AgentEvent::TurnEnd { .. }+ | AgentEvent::ToolExecutionUpdate { .. }+ | AgentEvent::ToolExecutionProgress { .. } => {}}}diff --git a/crates/pa-core/src/session_engine/tests.rs b/crates/pa-core/src/session_engine/tests.rsindex 520fcac9f6..2d3a693a9c 100644--- a/crates/pa-core/src/session_engine/tests.rs+++ b/crates/pa-core/src/session_engine/tests.rs@@ -826,6 +826,352 @@ async fn prompt_persists_tool_results() {assert!(entry["timestamp"].as_str().is_some());}+/// The tool-call lifecycle trace: a `tool_started` row at call issuance+/// (call id, tool name, the command text), `tool_progress` heartbeat ticks+/// while a long call runs, and the completed result carrying its duration.+/// A stalled call is visible in the session trace while it runs — the+/// Oscar incident showed a 13-minute recursive grep with a blank trace+/// between issuance and the user's cancel.+#[tokio::test]+async fn long_tool_call_persists_started_tick_and_duration_rows() {+ struct SlowTool;+ impl pa_agent::types::AgentTool for SlowTool {+ fn name(&self) -> &'static str {+ "slow"+ }+ fn description(&self) -> &'static str {+ "sleeps before answering"+ }+ fn parameters(&self) -> &serde_json::Value {+ static PARAMETERS: std::sync::OnceLock<serde_json::Value> = std::sync::OnceLock::new();+ PARAMETERS.get_or_init(|| serde_json::json!({ "type": "object" }))+ }+ fn execute(+ self: Arc<Self>,+ _tool_call_id: String,+ _params: serde_json::Value,+ _signal: pa_agent::abort::AbortSignal,+ _on_update: pa_agent::types::AgentToolUpdateCallback,+ ) -> pa_agent::BoxFut<'static, anyhow::Result<pa_agent::types::AgentToolResult>> {+ Box::pin(async {+ let started = std::time::Instant::now();+ // Staged long call: ~5s run with the tick threshold pinned+ // at 1s, so the heartbeat fires well before completion.+ tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ let mut result = pa_agent::types::AgentToolResult::text("done");+ result.details = serde_json::json!({+ "status": "ok",+ "durationMs": started.elapsed().as_millis() as u64,+ });+ Ok(result)+ })+ }+ }+ let provider = Arc::new(ScriptedProvider::new(test_model()));+ provider.push_tool_call_turn(+ Some("calling"),+ vec![(+ "call-1",+ "slow",+ serde_json::json!({ "command": "grep -rli training ~/" }),+ )],+ );+ provider.push_text_turn("done");+ let options = AgentOptions {+ initial_state: AgentInitialState {+ model: Some(test_model()),+ ..Default::default()+ },+ stream_fn: Some(provider.stream_fn()),+ tool_progress: Some(pa_agent::types::ToolProgressCadence {+ after_ms: 1_000,+ every_ms: 1_000,+ }),+ ..Default::default()+ };+ let agent = Agent::new(options);+ agent.set_tools(vec![Arc::new(SlowTool)]).await;+ let tmp = tempfile::tempdir().unwrap();+ let session = SessionManager::persisted(std::path::Path::new("/w"), tmp.path());+ let engine = AgentSession::new(Arc::new(agent), session, vec![])+ .await+ .unwrap();+ engine.prompt("hi", PromptOptions::default()).await.unwrap();+ engine.agent().wait_for_idle().await;+ let entries = engine.entries().await;++ let started = entries+ .iter()+ .find_map(|entry| match entry {+ FileEntry::ToolStarted { payload, .. } => Some(payload.clone()),+ _ => None,+ })+ .expect("tool_started row persisted");+ assert_eq!(started.tool_call_id, "call-1");+ assert_eq!(started.tool_name, "slow");+ assert_eq!(+ started.args,+ serde_json::json!({ "command": "grep -rli training ~/" })+ );++ let ticks: Vec<pa_types::session::ToolProgressEntry> = entries+ .iter()+ .filter_map(|entry| match entry {+ FileEntry::ToolProgress { payload, .. } => Some(payload.clone()),+ _ => None,+ })+ .collect();+ assert!(+ !ticks.is_empty(),+ "a 5s call with a 1s tick must emit at least one tool_progress row"+ );+ assert!(ticks.len() <= 5, "ticks bounded by the cadence: {ticks:?}");+ assert!(ticks.iter().all(|tick| tick.tool_call_id == "call-1"));+ assert!(ticks.iter().all(|tick| tick.tool_name == "slow"));+ assert!(+ ticks+ .iter()+ .all(|tick| tick.elapsed_ms >= 900 && tick.elapsed_ms < 5_500),+ "tick elapsed is wall-clock time since the call started: {ticks:?}"+ );++ let tool_result = entries+ .iter()+ .find_map(|entry| match entry {+ FileEntry::Message {+ message: SessionAgentMessage::ToolResult(result),+ ..+ } => Some(result.clone()),+ _ => None,+ })+ .expect("toolResult entry persisted");+ assert!(!tool_result.is_error);+ assert_eq!(+ tool_result.details.as_ref().and_then(|d| d.get("status")),+ Some(&serde_json::json!("ok"))+ );+ let duration = tool_result+ .details+ .as_ref()+ .and_then(|d| d.get("durationMs"))+ .and_then(serde_json::Value::as_u64)+ .expect("completed result carries durationMs");+ assert!(duration >= 4_500, "durationMs: {duration}");++ // The rows also live on disk in the session jsonl (the trace the+ // session zip contains), with the new types round-tripping.+ let file = tmp+ .path()+ .join(format!("{}.jsonl", engine.session_id().await))+ .to_string_lossy()+ .to_string();+ let lines: Vec<serde_json::Value> = std::fs::read_to_string(file)+ .unwrap()+ .lines()+ .map(|line| serde_json::from_str(line).unwrap())+ .collect();+ let started_line = lines+ .iter()+ .find(|line| line.get("type") == Some(&serde_json::json!("tool_started")))+ .expect("tool_started line on disk");+ assert_eq!(started_line["toolCallId"], "call-1");+ assert_eq!(started_line["toolName"], "slow");+ assert_eq!(started_line["args"]["command"], "grep -rli training ~/");+ assert!(+ lines+ .iter()+ .any(|line| line.get("type") == Some(&serde_json::json!("tool_progress"))),+ "tool_progress line on disk"+ );+}++/// The short-call contract: the start row and the result row persist, and+/// the heartbeat stays quiet (no `tool_progress` row under the threshold).+#[tokio::test]+async fn short_tool_call_persists_started_row_without_progress() {+ struct FastTool;+ impl pa_agent::types::AgentTool for FastTool {+ fn name(&self) -> &'static str {+ "fast"+ }+ fn description(&self) -> &'static str {+ "answers immediately"+ }+ fn parameters(&self) -> &serde_json::Value {+ static PARAMETERS: std::sync::OnceLock<serde_json::Value> = std::sync::OnceLock::new();+ PARAMETERS.get_or_init(|| serde_json::json!({ "type": "object" }))+ }+ fn execute(+ self: Arc<Self>,+ _tool_call_id: String,+ _params: serde_json::Value,+ _signal: pa_agent::abort::AbortSignal,+ _on_update: pa_agent::types::AgentToolUpdateCallback,+ ) -> pa_agent::BoxFut<'static, anyhow::Result<pa_agent::types::AgentToolResult>> {+ Box::pin(async { Ok(pa_agent::types::AgentToolResult::text("done")) })+ }+ }+ let provider = Arc::new(ScriptedProvider::new(test_model()));+ provider.push_tool_call_turn(+ Some("calling"),+ vec![("call-1", "fast", serde_json::json!({}))],+ );+ provider.push_text_turn("done");+ let options = AgentOptions {+ initial_state: AgentInitialState {+ model: Some(test_model()),+ ..Default::default()+ },+ stream_fn: Some(provider.stream_fn()),+ tool_progress: Some(pa_agent::types::ToolProgressCadence {+ after_ms: 1_000,+ every_ms: 1_000,+ }),+ ..Default::default()+ };+ let agent = Agent::new(options);+ agent.set_tools(vec![Arc::new(FastTool)]).await;+ let tmp = tempfile::tempdir().unwrap();+ let session = SessionManager::in_memory(tmp.path());+ let engine = AgentSession::new(Arc::new(agent), session, vec![])+ .await+ .unwrap();+ engine.prompt("hi", PromptOptions::default()).await.unwrap();+ engine.agent().wait_for_idle().await;+ let entries = engine.entries().await;+ assert!(+ entries.iter().any(|entry| matches!(entry, FileEntry::ToolStarted { payload, .. } if payload.tool_call_id == "call-1")),+ "short call still persists its tool_started row"+ );+ assert!(+ entries+ .iter()+ .all(|entry| !matches!(entry, FileEntry::ToolProgress { .. })),+ "a fast call must not emit tool_progress rows"+ );+ assert!(+ entries.iter().any(|entry| matches!(+ entry,+ FileEntry::Message {+ message: SessionAgentMessage::ToolResult(_),+ ..+ }+ )),+ "short call persists its toolResult row"+ );+}++/// The aborted completion carries the elapsed duration: the trace shows how+/// long the call ran before the kill, not just that it died. Before this+/// fix, the aborted toolResult row carried only the bare "Tool execution+/// aborted" text with no duration.+#[tokio::test]+async fn aborted_tool_call_result_carries_elapsed_duration() {+ struct StalledTool(Arc<std::sync::atomic::AtomicBool>);+ impl pa_agent::types::AgentTool for StalledTool {+ fn name(&self) -> &'static str {+ "stalled"+ }+ fn description(&self) -> &'static str {+ "never finishes on its own"+ }+ fn parameters(&self) -> &serde_json::Value {+ static PARAMETERS: std::sync::OnceLock<serde_json::Value> = std::sync::OnceLock::new();+ PARAMETERS.get_or_init(|| serde_json::json!({ "type": "object" }))+ }+ fn execute(+ self: Arc<Self>,+ _tool_call_id: String,+ _params: serde_json::Value,+ _signal: pa_agent::abort::AbortSignal,+ _on_update: pa_agent::types::AgentToolUpdateCallback,+ ) -> pa_agent::BoxFut<'static, anyhow::Result<pa_agent::types::AgentToolResult>> {+ let entered = Arc::clone(&self.0);+ Box::pin(async move {+ entered.store(true, std::sync::atomic::Ordering::SeqCst);+ // The recursive-scan shape: unbounded runtime, only the+ // abort ends it.+ tokio::time::sleep(std::time::Duration::from_secs(600)).await;+ Ok(pa_agent::types::AgentToolResult::text("done"))+ })+ }+ }+ let entered = Arc::new(std::sync::atomic::AtomicBool::new(false));+ let provider = Arc::new(ScriptedProvider::new(test_model()));+ provider.push_tool_call_turn(+ Some("calling"),+ vec![("call-1", "stalled", serde_json::json!({}))],+ );+ provider.push_text_turn("done");+ let options = AgentOptions {+ initial_state: AgentInitialState {+ model: Some(test_model()),+ ..Default::default()+ },+ stream_fn: Some(provider.stream_fn()),+ ..Default::default()+ };+ let agent = Agent::new(options);+ agent+ .set_tools(vec![Arc::new(StalledTool(Arc::clone(&entered)))])+ .await;+ let tmp = tempfile::tempdir().unwrap();+ let session = SessionManager::in_memory(tmp.path());+ let engine = Arc::new(+ AgentSession::new(Arc::new(agent), session, vec![])+ .await+ .unwrap(),+ );+ let prompt_engine = Arc::clone(&engine);+ let prompting =+ tokio::spawn(async move { prompt_engine.prompt("hi", PromptOptions::default()).await });+ // Bounded entered wait: a stalled tool that never starts fails the+ // test instead of hanging the suite.+ let tool_entered = tokio::time::timeout(std::time::Duration::from_secs(2), async {+ while !entered.load(std::sync::atomic::Ordering::SeqCst) {+ tokio::time::sleep(std::time::Duration::from_millis(10)).await;+ }+ })+ .await;+ assert!(+ tool_entered.is_ok(),+ "the stalled tool never entered within 2s"+ );+ // Let the call run briefly so the elapsed duration is non-zero.+ tokio::time::sleep(std::time::Duration::from_millis(150)).await;+ engine.agent().abort();+ prompting.await.unwrap().unwrap();+ engine.agent().wait_for_idle().await;+ let entries = engine.entries().await;+ let tool_result = entries+ .iter()+ .find_map(|entry| match entry {+ FileEntry::Message {+ message: SessionAgentMessage::ToolResult(result),+ ..+ } => Some(result.clone()),+ _ => None,+ })+ .expect("aborted toolResult entry persisted");+ assert!(tool_result.is_error);+ match &tool_result.content[..] {+ [pa_types::ai::UserContentBlock::Text(text)] => {+ assert_eq!(text.text, "Tool execution aborted");+ }+ other => panic!("expected the abort error text: {other:?}"),+ }+ let details = tool_result+ .details+ .as_ref()+ .expect("aborted result has details");+ assert_eq!(details.get("status"), Some(&serde_json::json!("aborted")));+ let duration = details+ .get("durationMs")+ .and_then(serde_json::Value::as_u64)+ .expect("aborted result carries durationMs");+ assert!(duration >= 100, "durationMs: {duration}");+}+/// Persisted `ipython`/`python` calls dispatch through `ToolDefinitionBridge`/// to the registered Python tool.#[tokio::test]diff --git a/crates/pa-daemon/src/agent_engine/turn/run_once.rs b/crates/pa-daemon/src/agent_engine/turn/run_once.rsindex 95b7354780..7777bb36cb 100644--- a/crates/pa-daemon/src/agent_engine/turn/run_once.rs+++ b/crates/pa-daemon/src/agent_engine/turn/run_once.rs@@ -295,6 +295,7 @@ impl AgentSessionEngine {args: args.clone(),});}+ AgentEvent::ToolExecutionProgress { .. } => {}AgentEvent::ToolExecutionUpdate {tool_call_id,partial_result,diff --git a/crates/pa-tui/src/tree_display.rs b/crates/pa-tui/src/tree_display.rsindex 9c9588f996..d1627e4b99 100644--- a/crates/pa-tui/src/tree_display.rs+++ b/crates/pa-tui/src/tree_display.rs@@ -297,6 +297,18 @@ pub fn entry_display_text<S: std::hash::BuildHasher + Default>(color(ThemeColor::Dim, "]".to_string()),]}+ FileEntry::ToolStarted { payload, .. } => vec![color(+ ThemeColor::Dim,+ format!("[tool started: {}]", payload.tool_name),+ )],+ FileEntry::ToolProgress { payload, .. } => vec![color(+ ThemeColor::Dim,+ format!(+ "[tool running: {} ({}s)]",+ payload.tool_name,+ payload.elapsed_ms / 1000+ ),+ )],FileEntry::Header { .. }| FileEntry::SessionState { .. }| FileEntry::GitState { .. }diff --git a/crates/pa-types/src/session.rs b/crates/pa-types/src/session.rsindex a9f0bfbb68..0f266c36b7 100644--- a/crates/pa-types/src/session.rs+++ b/crates/pa-types/src/session.rs@@ -300,6 +300,27 @@ pub struct GitStateEntry {pub git: GitContext,}+/// `type: "tool_started"`: a tool call issued to a tool (call id, tool name,+/// the call arguments) written when the call starts, so the trace shows an+/// in-flight call before its result row exists.+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct ToolStartedEntry {+ pub tool_call_id: String,+ pub tool_name: String,+ pub args: Value,+}++/// `type: "tool_progress"`: heartbeat row for a still-running tool call+/// (`elapsedMs` since the call started); quiet for short calls.+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct ToolProgressEntry {+ pub tool_call_id: String,+ pub tool_name: String,+ pub elapsed_ms: u64,+}+/// `type: "custom_message"`: a custom message preserved verbatim for replay/// (content/details/display round-trip through session restore).#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]@@ -337,6 +358,18 @@ pub enum FileEntry {#[serde(flatten)]base: EntryBase,},+ ToolStarted {+ #[serde(flatten)]+ payload: ToolStartedEntry,+ #[serde(flatten)]+ base: EntryBase,+ },+ ToolProgress {+ #[serde(flatten)]+ payload: ToolProgressEntry,+ #[serde(flatten)]+ base: EntryBase,+ },ThinkingLevelChange {#[serde(flatten)]payload: ThinkingLevelChangeEntry,@@ -436,6 +469,18 @@ enum KnownFileEntry {#[serde(flatten)]base: EntryBase,},+ ToolStarted {+ #[serde(flatten)]+ payload: ToolStartedEntry,+ #[serde(flatten)]+ base: EntryBase,+ },+ ToolProgress {+ #[serde(flatten)]+ payload: ToolProgressEntry,+ #[serde(flatten)]+ base: EntryBase,+ },ThinkingLevelChange {#[serde(flatten)]payload: ThinkingLevelChangeEntry,@@ -515,6 +560,8 @@ impl From<KnownFileEntry> for FileEntry {match entry {KnownFileEntry::Header { header } => Self::Header { header },KnownFileEntry::Message { message, base } => Self::Message { message, base },+ KnownFileEntry::ToolStarted { payload, base } => Self::ToolStarted { payload, base },+ KnownFileEntry::ToolProgress { payload, base } => Self::ToolProgress { payload, base },KnownFileEntry::ThinkingLevelChange { payload, base } => {Self::ThinkingLevelChange { payload, base }}@@ -565,6 +612,8 @@ impl FileEntry {return rest.get("id").and_then(|v| v.as_str());}FileEntry::Message { base, .. }+ | FileEntry::ToolStarted { base, .. }+ | FileEntry::ToolProgress { base, .. }| FileEntry::ThinkingLevelChange { base, .. }| FileEntry::ServiceTierChange { base, .. }| FileEntry::ModelChange { base, .. }@@ -590,6 +639,8 @@ impl FileEntry {return rest.get("parentId").and_then(|v| v.as_str());}FileEntry::Message { base, .. }+ | FileEntry::ToolStarted { base, .. }+ | FileEntry::ToolProgress { base, .. }| FileEntry::ThinkingLevelChange { base, .. }| FileEntry::ServiceTierChange { base, .. }| FileEntry::ModelChange { base, .. }@@ -618,6 +669,8 @@ impl FileEntry {.unwrap_or_default();}FileEntry::Message { base, .. }+ | FileEntry::ToolStarted { base, .. }+ | FileEntry::ToolProgress { base, .. }| FileEntry::ThinkingLevelChange { base, .. }| FileEntry::ServiceTierChange { base, .. }| FileEntry::ModelChange { base, .. }