Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(agent-session): record why a turn ended and whether it answered (#8400)

Wolf Mermelsteina9fa1fef6634
diff --git a/crates/agent/src/model/metering_http.rs b/crates/agent/src/model/metering_http.rs
index aa318e5f02..bdc98e90b8 100644
--- a/crates/agent/src/model/metering_http.rs
+++ b/crates/agent/src/model/metering_http.rs
@@ -1,7 +1,8 @@
//! Rig 0.41's public HTTP extension point. Its completion stream's inner stream
//! is private, and accepted-turn hooks miss invalid-tool recovery. Intercepting
//! each send here also accounts for `GenericEventSource` reconnect executions.
-//! No GenAI spans or aggregate analytics are emitted by this adapter.
+//! No GenAI spans or aggregate analytics are emitted by this adapter; it only
+//! adds the provider's stop reason to Rig's streaming span, which Rig drops.
use std::time::Duration;
@@ -11,6 +12,7 @@ use ai_usage::financial::{
};
use bytes::Bytes;
use futures::StreamExt;
+use genai_telemetry::{GenAiSpanExt as _, attr};
use rig_core::http_client::{
self, HttpClientExt, LazyBody, MultipartForm, Request, Response, StreamingResponse,
};
@@ -240,6 +242,9 @@ impl<H: HttpClientExt + Clone + 'static> HttpClientExt for MeteredHttpClient<H>
where
T: Into<Bytes> + Send,
{
+ // Rig sends from inside its `chat_streaming` span, so this is the
+ // model call the stop reason belongs to.
+ let call_span = tracing::Span::current();
let (request, attempt) = self.prepare(request.map(Into::into)).await?;
let Some(attempt) = attempt else {
return self.inner.send_streaming(request).await;
@@ -284,6 +289,12 @@ impl<H: HttpClientExt + Clone + 'static> HttpClientExt for MeteredHttpClient<H>
if facts.terminal
&& let Some(attempt) = attempt.take()
{
+ if let Some(reason) = &facts.stop_reason {
+ call_span.set_str_array(
+ attr::RESPONSE_FINISH_REASONS,
+ [reason.as_str()],
+ );
+ }
let pricing = attempt
.record_cost(facts.evidence(), facts.delivered_speed());
let finalized = attempt
@@ -384,6 +395,9 @@ struct Facts {
request_id: Option<ProviderRequestId>,
usage: Option<Value>,
service_tier: Option<String>,
+ /// Why the provider stopped generating, in its own words (`end_turn`,
+ /// `refusal`, `max_output_tokens`, `STOP`, ...).
+ stop_reason: Option<String>,
terminal: bool,
invalid: bool,
outcome: ProviderOutcome,
@@ -401,6 +415,7 @@ impl Facts {
request_id,
usage: None,
service_tier: None,
+ stop_reason: None,
terminal: false,
invalid: false,
outcome: ProviderOutcome::Succeeded,
@@ -469,6 +484,7 @@ impl Facts {
}
}
Some("message_delta") => {
+ self.note_stop_reason(value.pointer("/delta/stop_reason"));
// Cumulative counters, not deltas. Some endpoints include
// zero input/cache placeholders here: preserve start counts.
if let Some(update) = value.get("usage").and_then(Value::as_object)
@@ -504,6 +520,11 @@ impl Facts {
.and_then(Value::as_str)
.map(str::to_owned);
self.usage = value.pointer("/response/usage").cloned();
+ self.note_stop_reason(
+ value
+ .pointer("/response/incomplete_details/reason")
+ .or_else(|| value.pointer("/response/status")),
+ );
self.terminal = true;
if value["type"] == "response.failed" {
self.outcome = ProviderOutcome::Failed;
@@ -512,6 +533,7 @@ impl Facts {
}
WireProtocol::ChatCompletions => {
self.identify(&value);
+ self.note_stop_reason(value.pointer("/choices/0/finish_reason"));
// The include_usage event is a cumulative, final snapshot. A
// repeated snapshot is never added a second time.
if value.get("usage").is_some_and(Value::is_object) {
@@ -527,7 +549,8 @@ impl Facts {
if let Some(usage) = value.get("usageMetadata") {
self.usage = Some(usage.clone());
}
- if value.pointer("/candidates/0/finishReason").is_some() {
+ if let Some(reason) = value.pointer("/candidates/0/finishReason") {
+ self.note_stop_reason(Some(reason));
// Earlier chunks are cumulative but not necessarily complete.
self.usage = value.get("usageMetadata").cloned();
self.terminal = true;
@@ -536,6 +559,12 @@ impl Facts {
}
}
+ fn note_stop_reason(&mut self, reason: Option<&Value>) {
+ if let Some(reason) = reason.and_then(Value::as_str) {
+ self.stop_reason = Some(reason.to_owned());
+ }
+ }
+
fn delivered_speed(&self) -> Option<&str> {
if self.protocol == WireProtocol::Anthropic {
self.usage
diff --git a/crates/agent/src/model/metering_http/test.rs b/crates/agent/src/model/metering_http/test.rs
index f459de388d..43be05988f 100644
--- a/crates/agent/src/model/metering_http/test.rs
+++ b/crates/agent/src/model/metering_http/test.rs
@@ -792,6 +792,89 @@ fn incomplete_stream_snapshots_are_never_promoted_to_final_usage() {
assert!(matches!(chat.evidence(), UsageEvidence::Missing(_)));
}
+#[test]
+fn every_protocol_reports_why_the_provider_stopped() {
+ let mut anthropic = Facts::new(WireProtocol::Anthropic, false, None);
+ anthropic.event(
+ br#"{"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1}}}"#,
+ );
+ anthropic.event(
+ br#"{"type":"message_delta","delta":{"stop_reason":"refusal"},"usage":{"output_tokens":0}}"#,
+ );
+ anthropic.event(br#"{"type":"message_stop"}"#);
+ assert_eq!(anthropic.stop_reason.as_deref(), Some("refusal"));
+
+ let mut responses = Facts::new(WireProtocol::Responses, false, None);
+ responses.event(br#"{"type":"response.incomplete","response":{"status":"incomplete","incomplete_details":{"reason":"max_output_tokens"}}}"#);
+ assert_eq!(responses.stop_reason.as_deref(), Some("max_output_tokens"));
+
+ let mut completed = Facts::new(WireProtocol::Responses, false, None);
+ completed.event(br#"{"type":"response.completed","response":{"status":"completed"}}"#);
+ assert_eq!(completed.stop_reason.as_deref(), Some("completed"));
+
+ let mut chat = Facts::new(WireProtocol::ChatCompletions, false, None);
+ chat.event(br#"{"choices":[{"delta":{"content":"hi"},"finish_reason":null}]}"#);
+ chat.event(br#"{"choices":[{"delta":{},"finish_reason":"length"}]}"#);
+ chat.event(br#"{"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":2}}"#);
+ assert_eq!(chat.stop_reason.as_deref(), Some("length"));
+
+ let mut gemini = Facts::new(WireProtocol::Gemini, false, None);
+ gemini.event(br#"{"candidates":[{"finishReason":"SAFETY"}]}"#);
+ assert_eq!(gemini.stop_reason.as_deref(), Some("SAFETY"));
+}
+
+/// Rig's streaming span keeps only usage; the stop reason comes from here.
+#[tokio::test]
+async fn the_stop_reason_lands_on_the_model_call_span() {
+ use opentelemetry::trace::TracerProvider as _;
+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};
+ use tracing::Instrument as _;
+ use tracing_subscriber::layer::SubscriberExt as _;
+
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));
+ let _guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ tracing::callsite::rebuild_interest_cache();
+
+ let journal = Arc::new(Journal::default());
+ let sse = concat!(
+ "data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg\",\"usage\":{\"input_tokens\":10,\"output_tokens\":1}}}\n\n",
+ "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"refusal\"},\"usage\":{\"output_tokens\":0}}\n\n",
+ "data: {\"type\":\"message_stop\"}\n\n"
+ );
+ let client = MeteredHttpClient::new(Transport::new(sse), "anthropic", WireProtocol::Anthropic);
+ context(
+ journal.clone(),
+ WireProtocol::Anthropic,
+ "anthropic",
+ "model",
+ )
+ .scope(
+ async {
+ let mut body = client.send_streaming(request()).await.unwrap().into_body();
+ while body.next().await.is_some() {}
+ }
+ .instrument(tracing::info_span!("chat_streaming")),
+ )
+ .await;
+
+ provider.force_flush().unwrap();
+ let spans = exporter.get_finished_spans().unwrap();
+ let call = spans
+ .iter()
+ .find(|span| span.name == "chat_streaming")
+ .expect("the model call span finished");
+ let reasons = call
+ .attributes
+ .iter()
+ .find(|kv| kv.key.as_str() == attr::RESPONSE_FINISH_REASONS)
+ .map(|kv| kv.value.to_string());
+ assert_eq!(reasons.as_deref(), Some("[\"refusal\"]"));
+}
+
#[tokio::test]
async fn replayed_admission_never_executes_again() {
let journal = Arc::new(Journal {
diff --git a/crates/agent_inmem/src/domain/agent.rs b/crates/agent_inmem/src/domain/agent.rs
index e82ae56d2d..09c635b720 100644
--- a/crates/agent_inmem/src/domain/agent.rs
+++ b/crates/agent_inmem/src/domain/agent.rs
@@ -653,6 +653,10 @@ pub async fn serve(state: Arc<AgentState>, acp: AcpChannel) -> Result<(), AcpErr
agent.turn.first_text_ms = tracing::field::Empty,
agent.turn.admission_wait_ms = tracing::field::Empty,
agent.turn.lock_wait_ms = tracing::field::Empty,
+ agent.turn.stop_reason = tracing::field::Empty,
+ agent.turn.text_chars = tracing::field::Empty,
+ agent.turn.tool_calls = tracing::field::Empty,
+ agent.turn.replied = tracing::field::Empty,
);
genai_telemetry::propagation::set_parent(&span, request.meta.as_ref());
let prompt = UserPrompt::from_request(&request);
@@ -947,6 +951,7 @@ async fn run_turn(
let mut was_cancelled = false;
let mut first_part_seen = false;
let mut first_text_seen = false;
+ let mut output = TurnOutput::default();
loop {
match tokio::time::timeout(TURN_IDLE_TIMEOUT, parts.recv()).await {
Ok(Some(Ok(part))) => {
@@ -978,6 +983,7 @@ async fn run_turn(
break;
}
}
+ output.observe(&part);
accumulator.push(part);
}
Ok(Some(Err(error))) => {
@@ -1026,12 +1032,60 @@ async fn run_turn(
// A failure first: the idle timeout cancels the stream itself, and a turn
// it stopped failed rather than being stopped by the user.
- if let Some(failure) = failure {
+ let result = if let Some(failure) = failure {
Err(AcpError::internal_error().data(failure))
} else if was_cancelled || cancel.is_cancelled() {
Ok(StopReason::Cancelled)
} else {
Ok(StopReason::EndTurn)
+ };
+ output.record(&span, &result);
+ result
+}
+
+/// What a turn said, recorded on its `agent.acp.prompt` span so a turn that
+/// ended without answering is found by query, not by reading its log.
+#[derive(Default)]
+struct TurnOutput {
+ text_chars: i64,
+ tool_calls: i64,
+ /// Text after the last tool call: the answer the reader is left with.
+ replied: bool,
+}
+
+impl TurnOutput {
+ fn observe(&mut self, part: &StreamPart) {
+ match part {
+ StreamPart::Content(text) => {
+ self.text_chars += i64::try_from(text.chars().count()).unwrap_or(i64::MAX);
+ self.replied |= !text.trim().is_empty();
+ }
+ StreamPart::ToolCall(_) => {
+ self.tool_calls += 1;
+ self.replied = false;
+ }
+ StreamPart::Thinking(_) | StreamPart::ToolResponse(_) | StreamPart::Usage(_) => {}
+ }
+ }
+
+ // `i64`, not `u64`: a `u64` reaches OpenTelemetry as a string attribute.
+ fn record(&self, span: &tracing::Span, result: &Result<StopReason, AcpError>) {
+ let stop_reason = match result {
+ Ok(StopReason::EndTurn) => "end_turn",
+ Ok(StopReason::Cancelled) => "cancelled",
+ Ok(_) => "other",
+ Err(_) => "error",
+ };
+ span.record("agent.turn.stop_reason", stop_reason);
+ span.record("agent.turn.text_chars", self.text_chars);
+ span.record("agent.turn.tool_calls", self.tool_calls);
+ span.record("agent.turn.replied", self.replied);
+ if stop_reason == "end_turn" && !self.replied {
+ tracing::warn!(
+ tool_calls = self.tool_calls,
+ "the turn ended without a reply after its last tool call"
+ );
+ }
}
}
diff --git a/crates/agent_inmem/src/domain/agent/test.rs b/crates/agent_inmem/src/domain/agent/test.rs
index e9f83a43d1..252be631f9 100644
--- a/crates/agent_inmem/src/domain/agent/test.rs
+++ b/crates/agent_inmem/src/domain/agent/test.rs
@@ -2677,3 +2677,22 @@ async fn changing_speed_applies_to_next_turn_and_rejects_unsupported_models() {
assert_eq!(requests[0].speed, agent::ModelSpeed::Ultrafast);
assert_eq!(requests[1].speed, agent::ModelSpeed::Standard);
}
+
+#[test]
+fn a_turn_that_ends_on_a_tool_call_has_not_replied() {
+ let mut output = TurnOutput::default();
+ output.observe(&StreamPart::Content("Let me look.".to_owned()));
+ output.observe(&StreamPart::ToolCall(agent::ToolCall {
+ id: "call-1".to_owned(),
+ name: "ExecuteCode".to_owned(),
+ json: serde_json::json!({}),
+ mcp: None,
+ }));
+ output.observe(&StreamPart::Content("\n\n".to_owned()));
+ assert_eq!(output.text_chars, 14);
+ assert_eq!(output.tool_calls, 1);
+ assert!(!output.replied);
+
+ output.observe(&StreamPart::Content("Found it.".to_owned()));
+ assert!(output.replied);
+}
diff --git a/crates/agent_session/README.md b/crates/agent_session/README.md
index ce27318d97..f117ade0cb 100644
--- a/crates/agent_session/README.md
+++ b/crates/agent_session/README.md
@@ -76,6 +76,19 @@ prompt spans also split `agent.turn.lock_wait_ms`, `agent.turn.admission_wait_ms
and `agent.turn.first_text_ms`; their legacy `ttft_ms` can include reasoning or
a tool call, but excludes usage and tool-result events.
+### A turn that ended without an answer
+
+In-memory prompt spans record what the turn said: `agent.turn.stop_reason`
+(`end_turn`, `cancelled`, `error`), `agent.turn.text_chars`,
+`agent.turn.tool_calls`, and `agent.turn.replied`, which is false when no text
+followed the last tool call. Such a turn also logs `the turn ended without a
+reply after its last tool call`; query `@agent.turn.replied:false` to find them.
+Then read the turn's last `chat_streaming` span: `gen_ai.response.finish_reasons`
+carries the provider's own stop reason (`end_turn`, `refusal`, `max_tokens`,
+`max_output_tokens`, `SAFETY`, ...), which Rig does not record for streamed
+calls. Each `execute_tool` span carries `macro.genai.tool.result_bytes`, whatever
+the content policy, so a result large enough to crowd the context shows there.
+
### From the send click to the first painted text
The web app's `agent.prompt` span (service `web-app`) covers one prompt end to
diff --git a/crates/agent_session/src/domain/session/telemetry.rs b/crates/agent_session/src/domain/session/telemetry.rs
index 698b49c521..7498259ebc 100644
--- a/crates/agent_session/src/domain/session/telemetry.rs
+++ b/crates/agent_session/src/domain/session/telemetry.rs
@@ -831,6 +831,16 @@ impl GenAiProjector {
error = Some(format!("the command exited with code {code}"));
}
+ if let Some(result) = &result {
+ let bytes = match result {
+ Value::String(text) => text.len(),
+ other => other.to_string().len(),
+ };
+ tool.span.set_u64(
+ attr::MACRO_TOOL_RESULT_BYTES,
+ u64::try_from(bytes).unwrap_or(u64::MAX),
+ );
+ }
if self.policy.capture {
if let Some(arguments) = &tool.arguments {
let (json, truncated) = bounded_json_string(arguments, &self.policy.limits);
diff --git a/crates/agent_session/src/domain/session/telemetry/test.rs b/crates/agent_session/src/domain/session/telemetry/test.rs
index 405331751b..0f53828710 100644
--- a/crates/agent_session/src/domain/session/telemetry/test.rs
+++ b/crates/agent_session/src/domain/session/telemetry/test.rs
@@ -185,6 +185,10 @@ fn a_turn_becomes_an_agent_span_with_a_tool_span_per_call() {
result.contains("events.rs"),
"terminal output recorded: {result}"
);
+ assert_eq!(
+ int_attribute(bash, attr::MACRO_TOOL_RESULT_BYTES),
+ Some(i64::try_from(result.len()).unwrap())
+ );
assert_eq!(bash.status, Status::Unset);
let write = spans_named(&spans, "execute_tool Write");
@@ -393,6 +397,8 @@ fn content_is_not_recorded_when_capture_is_off() {
let bash = spans_named(&spans, "execute_tool Bash")[0];
assert!(attribute(bash, attr::TOOL_CALL_ARGUMENTS).is_none());
assert!(attribute(bash, attr::TOOL_CALL_RESULT).is_none());
+ // A size is not content: it is recorded either way.
+ assert!(int_attribute(bash, attr::MACRO_TOOL_RESULT_BYTES).is_some_and(|bytes| bytes > 0));
assert_eq!(
string_attribute(bash, attr::TOOL_NAME).as_deref(),
Some("Bash")
diff --git a/crates/genai_telemetry/src/attr.rs b/crates/genai_telemetry/src/attr.rs
index 9853f4c18d..46b1c4e998 100644
--- a/crates/genai_telemetry/src/attr.rs
+++ b/crates/genai_telemetry/src/attr.rs
@@ -97,6 +97,10 @@ pub const MACRO_CHAT_FIRST_CHUNK_KIND: &str = "macro.genai.chat.first_chunk_kind
pub const MACRO_TOOL_KIND: &str = "macro.genai.tool.kind";
/// Macro-specific: the human-readable title the harness gave a tool call.
pub const MACRO_TOOL_TITLE: &str = "macro.genai.tool.title";
+/// Macro-specific: the size of a tool's whole result in bytes (a string
+/// result's text, anything else's JSON), recorded whether or not content is,
+/// so an oversized result shows without the result itself.
+pub const MACRO_TOOL_RESULT_BYTES: &str = "macro.genai.tool.result_bytes";
/// Macro-specific: the MCP server a tool was reached through, when it was.
pub const MACRO_MCP_SERVER: &str = "macro.genai.mcp.server";