Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(agent): stream thinking, and record how the provider stream behaved (#6788)
Wolf Mermelstein85155de51c1d
diff --git a/crates/agent/src/agent_loop.rs b/crates/agent/src/agent_loop.rsindex 243e63e95c..47cc8048fd 100644--- a/crates/agent/src/agent_loop.rs+++ b/crates/agent/src/agent_loop.rs@@ -396,6 +396,9 @@ impl Session {&mut self,messages: Vec<Message>,) -> Result<ChatCompletionStream<'_>, AgentError> {+ // `agent.stream.*` are recorded by the stream driver as the run ends;+ // see `StreamLiveness` in `crate::model::router`. Declared on both+ // shapes of the span because either one can be the run's.let span = if self.telemetry.enabled() {tracing::info_span!("invoke_agent",@@ -404,9 +407,18 @@ impl Session {gen_ai.conversation.id = self.telemetry.conversation_id(),gen_ai.provider.name = self.telemetry.provider_name(),gen_ai.request.model = self.telemetry.model_name(),+ agent.stream.items = tracing::field::Empty,+ agent.stream.first_item_ms = tracing::field::Empty,+ agent.stream.trailing_silence_ms = tracing::field::Empty,)} else {- tracing::info_span!("agent.turn", agent.name = %self.telemetry.agent_name())+ tracing::info_span!(+ "agent.turn",+ agent.name = %self.telemetry.agent_name(),+ agent.stream.items = tracing::field::Empty,+ agent.stream.first_item_ms = tracing::field::Empty,+ agent.stream.trailing_silence_ms = tracing::field::Empty,+ )};let telemetry = self.telemetry.clone();let result = selfdiff --git a/crates/agent/src/model/router.rs b/crates/agent/src/model/router.rsindex fad7b8daa1..0ce53a704d 100644--- a/crates/agent/src/model/router.rs+++ b/crates/agent/src/model/router.rs@@ -17,6 +17,7 @@use std::collections::HashMap;use std::sync::{Arc, OnceLock};+use std::time::{Duration, Instant};use ai_toolset::RequestContext;use ai_usage::{UsageContext, UsageRecorder};@@ -501,6 +502,67 @@ whereself.telemetry.finish_run();}}+ /// Liveness of the provider stream, recorded on the run's span by `Drop` so+ /// it lands however the driver ends - including the abort a cancelled turn+ /// causes, which is the case most worth seeing.+ ///+ /// These answer what a parked stream cannot: the task is idle whether the+ /// provider is dribbling tokens or has gone silent, and only the timing of+ /// the items tells those apart. Read `trailing_silence_ms` first - near+ /// zero means the model was still producing when the run ended, a long tail+ /// means it had stopped talking to us well before.+ struct StreamLiveness {+ span: tracing::Span,+ started_at: Instant,+ items: i64,+ first_item: Option<Duration>,+ last_item: Option<Duration>,+ }++ impl StreamLiveness {+ fn new(span: tracing::Span) -> Self {+ Self {+ span,+ started_at: Instant::now(),+ items: 0,+ first_item: None,+ last_item: None,+ }+ }++ fn observed(&mut self) {+ let at = self.started_at.elapsed();+ self.items += 1;+ self.first_item.get_or_insert(at);+ self.last_item = Some(at);+ }+ }++ impl Drop for StreamLiveness {+ // Recorded as `i64`: a `u64` reaches OpenTelemetry as a *string*+ // attribute, which every numeric query would then silently miss.+ fn drop(&mut self) {+ self.span.record("agent.stream.items", self.items);+ if let Some(first) = self.first_item {+ self.span+ .record("agent.stream.first_item_ms", millis(first));+ }+ // With nothing ever received the silence is the whole run, which is+ // what `unwrap_or_default` says here.+ let silent_since = self.last_item.unwrap_or_default();+ self.span.record(+ "agent.stream.trailing_silence_ms",+ millis(self.started_at.elapsed().saturating_sub(silent_since)),+ );+ }+ }++ /// Whole milliseconds, saturating: a duration longer than `i64::MAX`+ /// milliseconds is not a number anybody needs exactly.+ fn millis(duration: Duration) -> i64 {+ i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)+ }+let mut finish_run = FinishRun {telemetry: telemetry.clone(),agent_span: agent_span.clone(),@@ -509,20 +571,22 @@ wherelet driver_span = agent_span.clone();let driver = tokio::spawn(async move {- let mut thinking_buf = String::new();+ let mut liveness = StreamLiveness::new(agent_span.clone());while let Some(item) = rig_stream.next().await {+ liveness.observed();match item {Ok(MultiTurnStreamItem::StreamAssistantItem(StreamedAssistantContent::ReasoningDelta { reasoning, .. },)) => {- thinking_buf.push_str(&reasoning);+ // Forwarded as it arrives, like a text delta. Held back+ // until the model says something else, a turn that+ // thinks for half a minute before its first tool call+ // shows the reader nothing for that whole time and then+ // the entire thought at once.+ let _ = driver_tx.send(Ok(StreamPart::Thinking(reasoning)));}other => {- if !thinking_buf.is_empty() {- let _ = driver_tx- .send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));- }match other {Ok(MultiTurnStreamItem::FinalResponse(final_resp)) => {let usage = final_resp.usage;@@ -573,9 +637,6 @@ where}}}- if !thinking_buf.is_empty() {- let _ = driver_tx.send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));- }// Dropping `rig_stream` (and with it the hook's sender) plus `driver_tx`// here closes the channel, ending the consumer stream below.drop(finish_run);diff --git a/crates/agent/src/test/agent_loop/mod.rs b/crates/agent/src/test/agent_loop/mod.rsindex 6816d89245..6ae4f8b1a6 100644--- a/crates/agent/src/test/agent_loop/mod.rs+++ b/crates/agent/src/test/agent_loop/mod.rs@@ -8,6 +8,7 @@ mod util;mod test_cooperative_cancellation;mod test_eager_tools;mod test_telemetry;+mod test_thinking;mod test_tool;mod test_tool_search;mod test_user_tools;diff --git a/crates/agent/src/test/agent_loop/test_telemetry.rs b/crates/agent/src/test/agent_loop/test_telemetry.rsindex f14a7f93ae..fe2ec68529 100644--- a/crates/agent/src/test/agent_loop/test_telemetry.rs+++ b/crates/agent/src/test/agent_loop/test_telemetry.rs@@ -586,3 +586,56 @@ async fn a_loop_with_telemetry_off_records_no_genai_content() {);}}++/// Stream liveness: how many items the provider sent, when the first arrived,+/// and how long it had been quiet when the run ended. Without these a parked+/// stream is unreadable - a provider dribbling tokens and one that has gone+/// silent look identical from the span's duration alone.+#[tokio::test]+async fn the_run_span_records_how_the_provider_stream_behaved() {+ let (exporter, provider, _guard) = otel_test_pipeline();++ let model = MockCompletionModel::from_stream_turns([vec![+ MockStreamEvent::ReasoningDelta {+ id: None,+ reasoning: "thinking".to_owned(),+ },+ MockStreamEvent::Text("done".to_owned()),+ MockStreamEvent::final_response_with_default_usage(),+ ]]);++ let mut session = util::test_loop()+ .with_genai_telemetry(false)+ .test_session(+ util::tool_set(ai_toolset::AsyncToolCollection::<()>::new()),+ Arc::new(()),+ "test preamble",+ util::usage_ctx(),+ model,+ )+ .await;+ util::drive(&mut session, "think then answer").await;+ drop(session);++ let spans = finished(&exporter, &provider);+ let run = spans+ .iter()+ .find(|span| span.name == "agent.turn")+ .unwrap_or_else(|| panic!("the run span, got {}", describe(&spans)));++ // Numbers, not strings: a `u64` reaches OpenTelemetry as a string+ // attribute and every numeric query on it silently misses.+ assert!(+ int_attribute(run, "agent.stream.items").is_some_and(|items| items >= 3),+ "every item the driver saw is counted, as a number: {:?}",+ attribute(run, "agent.stream.items")+ );+ assert!(+ int_attribute(run, "agent.stream.first_item_ms").is_some(),+ "the wait for the first item is recorded"+ );+ assert!(+ int_attribute(run, "agent.stream.trailing_silence_ms").is_some(),+ "the silence at the end is recorded"+ );+}diff --git a/crates/agent/src/test/agent_loop/test_thinking.rs b/crates/agent/src/test/agent_loop/test_thinking.rsnew file mode 100644index 0000000000..85e3d53ede--- /dev/null+++ b/crates/agent/src/test/agent_loop/test_thinking.rs@@ -0,0 +1,92 @@+//! Thinking reaches the consumer as it is produced.+//!+//! A reasoning model can think for tens of seconds before it says anything+//! else. Holding those deltas until the next non-reasoning item - a tool call,+//! text, the final response - shows the reader nothing for that whole stretch+//! and then the entire thought at once, which is the opposite of what a+//! thinking stream is for.++use super::util;+use crate::stream::StreamPart;+use ai_toolset::AsyncToolCollection;+use rig_core::message::Message;+use rig_core::test_utils::{MockCompletionModel, MockStreamEvent};+use std::sync::Arc;++fn reasoning(text: &str) -> MockStreamEvent {+ MockStreamEvent::ReasoningDelta {+ id: None,+ reasoning: text.to_owned(),+ }+}++/// Each delta arrives as its own part, in order. One coalesced `Thinking`+/// part would pass a "the thought came through" assertion just as well, which+/// is why this pins the shape rather than the text.+#[tokio::test]+async fn reasoning_deltas_stream_as_they_arrive() {+ let model = MockCompletionModel::from_stream_turns([vec![+ reasoning("first "),+ reasoning("second "),+ reasoning("third"),+ MockStreamEvent::final_response_with_default_usage(),+ ]]);++ let mut session = util::session(+ util::tool_set(AsyncToolCollection::<()>::new()),+ Arc::new(()),+ model,+ )+ .await;+ let collected = util::collect(+ session+ .send_message(vec![Message::user("think")])+ .await+ .expect("the stream starts"),+ )+ .await;++ let thoughts: Vec<&str> = collected+ .parts+ .iter()+ .filter_map(|part| match part {+ StreamPart::Thinking(text) => Some(text.as_str()),+ _ => None,+ })+ .collect();++ assert_eq!(thoughts, vec!["first ", "second ", "third"]);+ assert!(collected.error.is_none());+}++/// A turn that ends while still thinking - the model produced nothing else,+/// or the run was cancelled - has still shown its reader everything it+/// thought. Nothing waits on a final response that may never come.+#[tokio::test]+async fn thinking_reaches_the_consumer_without_a_following_item() {+ let model =+ MockCompletionModel::from_stream_turns([vec![reasoning("thought that stands alone")]]);++ let mut session = util::session(+ util::tool_set(AsyncToolCollection::<()>::new()),+ Arc::new(()),+ model,+ )+ .await;+ let collected = util::collect(+ session+ .send_message(vec![Message::user("think")])+ .await+ .expect("the stream starts"),+ )+ .await;++ assert!(+ collected.parts.iter().any(|part| matches!(+ part,+ StreamPart::Thinking(text) if text == "thought that stands alone"+ )),+ "the thought reached the consumer: {:?}",+ collected.parts+ );+}