Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(agent-session): record why a turn ended and whether it answered (#8400)
Wolf Mermelsteina9fa1fef6634
diff --git a/crates/agent/src/model/metering_http.rs b/crates/agent/src/model/metering_http.rsindex aa318e5f02..bdc98e90b8 100644--- a/crates/agent/src/model/metering_http.rs+++ b/crates/agent/src/model/metering_http.rs@@ -1,7 +1,8 @@//! Rig 0.41's public HTTP extension point. Its completion stream's inner stream//! is private, and accepted-turn hooks miss invalid-tool recovery. Intercepting//! each send here also accounts for `GenericEventSource` reconnect executions.-//! No GenAI spans or aggregate analytics are emitted by this adapter.+//! No GenAI spans or aggregate analytics are emitted by this adapter; it only+//! adds the provider's stop reason to Rig's streaming span, which Rig drops.use std::time::Duration;@@ -11,6 +12,7 @@ use ai_usage::financial::{};use bytes::Bytes;use futures::StreamExt;+use genai_telemetry::{GenAiSpanExt as _, attr};use rig_core::http_client::{self, HttpClientExt, LazyBody, MultipartForm, Request, Response, StreamingResponse,};@@ -240,6 +242,9 @@ impl<H: HttpClientExt + Clone + 'static> HttpClientExt for MeteredHttpClient<H>whereT: Into<Bytes> + Send,{+ // Rig sends from inside its `chat_streaming` span, so this is the+ // model call the stop reason belongs to.+ let call_span = tracing::Span::current();let (request, attempt) = self.prepare(request.map(Into::into)).await?;let Some(attempt) = attempt else {return self.inner.send_streaming(request).await;@@ -284,6 +289,12 @@ impl<H: HttpClientExt + Clone + 'static> HttpClientExt for MeteredHttpClient<H>if facts.terminal&& let Some(attempt) = attempt.take(){+ if let Some(reason) = &facts.stop_reason {+ call_span.set_str_array(+ attr::RESPONSE_FINISH_REASONS,+ [reason.as_str()],+ );+ }let pricing = attempt.record_cost(facts.evidence(), facts.delivered_speed());let finalized = attempt@@ -384,6 +395,9 @@ struct Facts {request_id: Option<ProviderRequestId>,usage: Option<Value>,service_tier: Option<String>,+ /// Why the provider stopped generating, in its own words (`end_turn`,+ /// `refusal`, `max_output_tokens`, `STOP`, ...).+ stop_reason: Option<String>,terminal: bool,invalid: bool,outcome: ProviderOutcome,@@ -401,6 +415,7 @@ impl Facts {request_id,usage: None,service_tier: None,+ stop_reason: None,terminal: false,invalid: false,outcome: ProviderOutcome::Succeeded,@@ -469,6 +484,7 @@ impl Facts {}}Some("message_delta") => {+ self.note_stop_reason(value.pointer("/delta/stop_reason"));// Cumulative counters, not deltas. Some endpoints include// zero input/cache placeholders here: preserve start counts.if let Some(update) = value.get("usage").and_then(Value::as_object)@@ -504,6 +520,11 @@ impl Facts {.and_then(Value::as_str).map(str::to_owned);self.usage = value.pointer("/response/usage").cloned();+ self.note_stop_reason(+ value+ .pointer("/response/incomplete_details/reason")+ .or_else(|| value.pointer("/response/status")),+ );self.terminal = true;if value["type"] == "response.failed" {self.outcome = ProviderOutcome::Failed;@@ -512,6 +533,7 @@ impl Facts {}WireProtocol::ChatCompletions => {self.identify(&value);+ self.note_stop_reason(value.pointer("/choices/0/finish_reason"));// The include_usage event is a cumulative, final snapshot. A// repeated snapshot is never added a second time.if value.get("usage").is_some_and(Value::is_object) {@@ -527,7 +549,8 @@ impl Facts {if let Some(usage) = value.get("usageMetadata") {self.usage = Some(usage.clone());}- if value.pointer("/candidates/0/finishReason").is_some() {+ if let Some(reason) = value.pointer("/candidates/0/finishReason") {+ self.note_stop_reason(Some(reason));// Earlier chunks are cumulative but not necessarily complete.self.usage = value.get("usageMetadata").cloned();self.terminal = true;@@ -536,6 +559,12 @@ impl Facts {}}+ fn note_stop_reason(&mut self, reason: Option<&Value>) {+ if let Some(reason) = reason.and_then(Value::as_str) {+ self.stop_reason = Some(reason.to_owned());+ }+ }+fn delivered_speed(&self) -> Option<&str> {if self.protocol == WireProtocol::Anthropic {self.usagediff --git a/crates/agent/src/model/metering_http/test.rs b/crates/agent/src/model/metering_http/test.rsindex f459de388d..43be05988f 100644--- a/crates/agent/src/model/metering_http/test.rs+++ b/crates/agent/src/model/metering_http/test.rs@@ -792,6 +792,89 @@ fn incomplete_stream_snapshots_are_never_promoted_to_final_usage() {assert!(matches!(chat.evidence(), UsageEvidence::Missing(_)));}+#[test]+fn every_protocol_reports_why_the_provider_stopped() {+ let mut anthropic = Facts::new(WireProtocol::Anthropic, false, None);+ anthropic.event(+ br#"{"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1}}}"#,+ );+ anthropic.event(+ br#"{"type":"message_delta","delta":{"stop_reason":"refusal"},"usage":{"output_tokens":0}}"#,+ );+ anthropic.event(br#"{"type":"message_stop"}"#);+ assert_eq!(anthropic.stop_reason.as_deref(), Some("refusal"));++ let mut responses = Facts::new(WireProtocol::Responses, false, None);+ responses.event(br#"{"type":"response.incomplete","response":{"status":"incomplete","incomplete_details":{"reason":"max_output_tokens"}}}"#);+ assert_eq!(responses.stop_reason.as_deref(), Some("max_output_tokens"));++ let mut completed = Facts::new(WireProtocol::Responses, false, None);+ completed.event(br#"{"type":"response.completed","response":{"status":"completed"}}"#);+ assert_eq!(completed.stop_reason.as_deref(), Some("completed"));++ let mut chat = Facts::new(WireProtocol::ChatCompletions, false, None);+ chat.event(br#"{"choices":[{"delta":{"content":"hi"},"finish_reason":null}]}"#);+ chat.event(br#"{"choices":[{"delta":{},"finish_reason":"length"}]}"#);+ chat.event(br#"{"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":2}}"#);+ assert_eq!(chat.stop_reason.as_deref(), Some("length"));++ let mut gemini = Facts::new(WireProtocol::Gemini, false, None);+ gemini.event(br#"{"candidates":[{"finishReason":"SAFETY"}]}"#);+ assert_eq!(gemini.stop_reason.as_deref(), Some("SAFETY"));+}++/// Rig's streaming span keeps only usage; the stop reason comes from here.+#[tokio::test]+async fn the_stop_reason_lands_on_the_model_call_span() {+ use opentelemetry::trace::TracerProvider as _;+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};+ use tracing::Instrument as _;+ use tracing_subscriber::layer::SubscriberExt as _;++ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));+ let _guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ tracing::callsite::rebuild_interest_cache();++ let journal = Arc::new(Journal::default());+ let sse = concat!(+ "data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg\",\"usage\":{\"input_tokens\":10,\"output_tokens\":1}}}\n\n",+ "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"refusal\"},\"usage\":{\"output_tokens\":0}}\n\n",+ "data: {\"type\":\"message_stop\"}\n\n"+ );+ let client = MeteredHttpClient::new(Transport::new(sse), "anthropic", WireProtocol::Anthropic);+ context(+ journal.clone(),+ WireProtocol::Anthropic,+ "anthropic",+ "model",+ )+ .scope(+ async {+ let mut body = client.send_streaming(request()).await.unwrap().into_body();+ while body.next().await.is_some() {}+ }+ .instrument(tracing::info_span!("chat_streaming")),+ )+ .await;++ provider.force_flush().unwrap();+ let spans = exporter.get_finished_spans().unwrap();+ let call = spans+ .iter()+ .find(|span| span.name == "chat_streaming")+ .expect("the model call span finished");+ let reasons = call+ .attributes+ .iter()+ .find(|kv| kv.key.as_str() == attr::RESPONSE_FINISH_REASONS)+ .map(|kv| kv.value.to_string());+ assert_eq!(reasons.as_deref(), Some("[\"refusal\"]"));+}+#[tokio::test]async fn replayed_admission_never_executes_again() {let journal = Arc::new(Journal {diff --git a/crates/agent_inmem/src/domain/agent.rs b/crates/agent_inmem/src/domain/agent.rsindex e82ae56d2d..09c635b720 100644--- a/crates/agent_inmem/src/domain/agent.rs+++ b/crates/agent_inmem/src/domain/agent.rs@@ -653,6 +653,10 @@ pub async fn serve(state: Arc<AgentState>, acp: AcpChannel) -> Result<(), AcpErragent.turn.first_text_ms = tracing::field::Empty,agent.turn.admission_wait_ms = tracing::field::Empty,agent.turn.lock_wait_ms = tracing::field::Empty,+ agent.turn.stop_reason = tracing::field::Empty,+ agent.turn.text_chars = tracing::field::Empty,+ agent.turn.tool_calls = tracing::field::Empty,+ agent.turn.replied = tracing::field::Empty,);genai_telemetry::propagation::set_parent(&span, request.meta.as_ref());let prompt = UserPrompt::from_request(&request);@@ -947,6 +951,7 @@ async fn run_turn(let mut was_cancelled = false;let mut first_part_seen = false;let mut first_text_seen = false;+ let mut output = TurnOutput::default();loop {match tokio::time::timeout(TURN_IDLE_TIMEOUT, parts.recv()).await {Ok(Some(Ok(part))) => {@@ -978,6 +983,7 @@ async fn run_turn(break;}}+ output.observe(&part);accumulator.push(part);}Ok(Some(Err(error))) => {@@ -1026,12 +1032,60 @@ async fn run_turn(// A failure first: the idle timeout cancels the stream itself, and a turn// it stopped failed rather than being stopped by the user.- if let Some(failure) = failure {+ let result = if let Some(failure) = failure {Err(AcpError::internal_error().data(failure))} else if was_cancelled || cancel.is_cancelled() {Ok(StopReason::Cancelled)} else {Ok(StopReason::EndTurn)+ };+ output.record(&span, &result);+ result+}++/// What a turn said, recorded on its `agent.acp.prompt` span so a turn that+/// ended without answering is found by query, not by reading its log.+#[derive(Default)]+struct TurnOutput {+ text_chars: i64,+ tool_calls: i64,+ /// Text after the last tool call: the answer the reader is left with.+ replied: bool,+}++impl TurnOutput {+ fn observe(&mut self, part: &StreamPart) {+ match part {+ StreamPart::Content(text) => {+ self.text_chars += i64::try_from(text.chars().count()).unwrap_or(i64::MAX);+ self.replied |= !text.trim().is_empty();+ }+ StreamPart::ToolCall(_) => {+ self.tool_calls += 1;+ self.replied = false;+ }+ StreamPart::Thinking(_) | StreamPart::ToolResponse(_) | StreamPart::Usage(_) => {}+ }+ }++ // `i64`, not `u64`: a `u64` reaches OpenTelemetry as a string attribute.+ fn record(&self, span: &tracing::Span, result: &Result<StopReason, AcpError>) {+ let stop_reason = match result {+ Ok(StopReason::EndTurn) => "end_turn",+ Ok(StopReason::Cancelled) => "cancelled",+ Ok(_) => "other",+ Err(_) => "error",+ };+ span.record("agent.turn.stop_reason", stop_reason);+ span.record("agent.turn.text_chars", self.text_chars);+ span.record("agent.turn.tool_calls", self.tool_calls);+ span.record("agent.turn.replied", self.replied);+ if stop_reason == "end_turn" && !self.replied {+ tracing::warn!(+ tool_calls = self.tool_calls,+ "the turn ended without a reply after its last tool call"+ );+ }}}diff --git a/crates/agent_inmem/src/domain/agent/test.rs b/crates/agent_inmem/src/domain/agent/test.rsindex e9f83a43d1..252be631f9 100644--- a/crates/agent_inmem/src/domain/agent/test.rs+++ b/crates/agent_inmem/src/domain/agent/test.rs@@ -2677,3 +2677,22 @@ async fn changing_speed_applies_to_next_turn_and_rejects_unsupported_models() {assert_eq!(requests[0].speed, agent::ModelSpeed::Ultrafast);assert_eq!(requests[1].speed, agent::ModelSpeed::Standard);}++#[test]+fn a_turn_that_ends_on_a_tool_call_has_not_replied() {+ let mut output = TurnOutput::default();+ output.observe(&StreamPart::Content("Let me look.".to_owned()));+ output.observe(&StreamPart::ToolCall(agent::ToolCall {+ id: "call-1".to_owned(),+ name: "ExecuteCode".to_owned(),+ json: serde_json::json!({}),+ mcp: None,+ }));+ output.observe(&StreamPart::Content("\n\n".to_owned()));+ assert_eq!(output.text_chars, 14);+ assert_eq!(output.tool_calls, 1);+ assert!(!output.replied);++ output.observe(&StreamPart::Content("Found it.".to_owned()));+ assert!(output.replied);+}diff --git a/crates/agent_session/README.md b/crates/agent_session/README.mdindex ce27318d97..f117ade0cb 100644--- a/crates/agent_session/README.md+++ b/crates/agent_session/README.md@@ -76,6 +76,19 @@ prompt spans also split `agent.turn.lock_wait_ms`, `agent.turn.admission_wait_msand `agent.turn.first_text_ms`; their legacy `ttft_ms` can include reasoning ora tool call, but excludes usage and tool-result events.+### A turn that ended without an answer++In-memory prompt spans record what the turn said: `agent.turn.stop_reason`+(`end_turn`, `cancelled`, `error`), `agent.turn.text_chars`,+`agent.turn.tool_calls`, and `agent.turn.replied`, which is false when no text+followed the last tool call. Such a turn also logs `the turn ended without a+reply after its last tool call`; query `@agent.turn.replied:false` to find them.+Then read the turn's last `chat_streaming` span: `gen_ai.response.finish_reasons`+carries the provider's own stop reason (`end_turn`, `refusal`, `max_tokens`,+`max_output_tokens`, `SAFETY`, ...), which Rig does not record for streamed+calls. Each `execute_tool` span carries `macro.genai.tool.result_bytes`, whatever+the content policy, so a result large enough to crowd the context shows there.+### From the send click to the first painted textThe web app's `agent.prompt` span (service `web-app`) covers one prompt end todiff --git a/crates/agent_session/src/domain/session/telemetry.rs b/crates/agent_session/src/domain/session/telemetry.rsindex 698b49c521..7498259ebc 100644--- a/crates/agent_session/src/domain/session/telemetry.rs+++ b/crates/agent_session/src/domain/session/telemetry.rs@@ -831,6 +831,16 @@ impl GenAiProjector {error = Some(format!("the command exited with code {code}"));}+ if let Some(result) = &result {+ let bytes = match result {+ Value::String(text) => text.len(),+ other => other.to_string().len(),+ };+ tool.span.set_u64(+ attr::MACRO_TOOL_RESULT_BYTES,+ u64::try_from(bytes).unwrap_or(u64::MAX),+ );+ }if self.policy.capture {if let Some(arguments) = &tool.arguments {let (json, truncated) = bounded_json_string(arguments, &self.policy.limits);diff --git a/crates/agent_session/src/domain/session/telemetry/test.rs b/crates/agent_session/src/domain/session/telemetry/test.rsindex 405331751b..0f53828710 100644--- a/crates/agent_session/src/domain/session/telemetry/test.rs+++ b/crates/agent_session/src/domain/session/telemetry/test.rs@@ -185,6 +185,10 @@ fn a_turn_becomes_an_agent_span_with_a_tool_span_per_call() {result.contains("events.rs"),"terminal output recorded: {result}");+ assert_eq!(+ int_attribute(bash, attr::MACRO_TOOL_RESULT_BYTES),+ Some(i64::try_from(result.len()).unwrap())+ );assert_eq!(bash.status, Status::Unset);let write = spans_named(&spans, "execute_tool Write");@@ -393,6 +397,8 @@ fn content_is_not_recorded_when_capture_is_off() {let bash = spans_named(&spans, "execute_tool Bash")[0];assert!(attribute(bash, attr::TOOL_CALL_ARGUMENTS).is_none());assert!(attribute(bash, attr::TOOL_CALL_RESULT).is_none());+ // A size is not content: it is recorded either way.+ assert!(int_attribute(bash, attr::MACRO_TOOL_RESULT_BYTES).is_some_and(|bytes| bytes > 0));assert_eq!(string_attribute(bash, attr::TOOL_NAME).as_deref(),Some("Bash")diff --git a/crates/genai_telemetry/src/attr.rs b/crates/genai_telemetry/src/attr.rsindex 9853f4c18d..46b1c4e998 100644--- a/crates/genai_telemetry/src/attr.rs+++ b/crates/genai_telemetry/src/attr.rs@@ -97,6 +97,10 @@ pub const MACRO_CHAT_FIRST_CHUNK_KIND: &str = "macro.genai.chat.first_chunk_kindpub const MACRO_TOOL_KIND: &str = "macro.genai.tool.kind";/// Macro-specific: the human-readable title the harness gave a tool call.pub const MACRO_TOOL_TITLE: &str = "macro.genai.tool.title";+/// Macro-specific: the size of a tool's whole result in bytes (a string+/// result's text, anything else's JSON), recorded whether or not content is,+/// so an oversized result shows without the result itself.+pub const MACRO_TOOL_RESULT_BYTES: &str = "macro.genai.tool.result_bytes";/// Macro-specific: the MCP server a tool was reached through, when it was.pub const MACRO_MCP_SERVER: &str = "macro.genai.mcp.server";