Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(agent): stream thinking, and record how the provider stream behaved (#6788)

Wolf Mermelstein85155de51c1d
diff --git a/crates/agent/src/agent_loop.rs b/crates/agent/src/agent_loop.rs
index 243e63e95c..47cc8048fd 100644
--- a/crates/agent/src/agent_loop.rs
+++ b/crates/agent/src/agent_loop.rs
@@ -396,6 +396,9 @@ impl Session {
&mut self,
messages: Vec<Message>,
) -> Result<ChatCompletionStream<'_>, AgentError> {
+ // `agent.stream.*` are recorded by the stream driver as the run ends;
+ // see `StreamLiveness` in `crate::model::router`. Declared on both
+ // shapes of the span because either one can be the run's.
let span = if self.telemetry.enabled() {
tracing::info_span!(
"invoke_agent",
@@ -404,9 +407,18 @@ impl Session {
gen_ai.conversation.id = self.telemetry.conversation_id(),
gen_ai.provider.name = self.telemetry.provider_name(),
gen_ai.request.model = self.telemetry.model_name(),
+ agent.stream.items = tracing::field::Empty,
+ agent.stream.first_item_ms = tracing::field::Empty,
+ agent.stream.trailing_silence_ms = tracing::field::Empty,
)
} else {
- tracing::info_span!("agent.turn", agent.name = %self.telemetry.agent_name())
+ tracing::info_span!(
+ "agent.turn",
+ agent.name = %self.telemetry.agent_name(),
+ agent.stream.items = tracing::field::Empty,
+ agent.stream.first_item_ms = tracing::field::Empty,
+ agent.stream.trailing_silence_ms = tracing::field::Empty,
+ )
};
let telemetry = self.telemetry.clone();
let result = self
diff --git a/crates/agent/src/model/router.rs b/crates/agent/src/model/router.rs
index fad7b8daa1..0ce53a704d 100644
--- a/crates/agent/src/model/router.rs
+++ b/crates/agent/src/model/router.rs
@@ -17,6 +17,7 @@
use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
+use std::time::{Duration, Instant};
use ai_toolset::RequestContext;
use ai_usage::{UsageContext, UsageRecorder};
@@ -501,6 +502,67 @@ where
self.telemetry.finish_run();
}
}
+ /// Liveness of the provider stream, recorded on the run's span by `Drop` so
+ /// it lands however the driver ends - including the abort a cancelled turn
+ /// causes, which is the case most worth seeing.
+ ///
+ /// These answer what a parked stream cannot: the task is idle whether the
+ /// provider is dribbling tokens or has gone silent, and only the timing of
+ /// the items tells those apart. Read `trailing_silence_ms` first - near
+ /// zero means the model was still producing when the run ended, a long tail
+ /// means it had stopped talking to us well before.
+ struct StreamLiveness {
+ span: tracing::Span,
+ started_at: Instant,
+ items: i64,
+ first_item: Option<Duration>,
+ last_item: Option<Duration>,
+ }
+
+ impl StreamLiveness {
+ fn new(span: tracing::Span) -> Self {
+ Self {
+ span,
+ started_at: Instant::now(),
+ items: 0,
+ first_item: None,
+ last_item: None,
+ }
+ }
+
+ fn observed(&mut self) {
+ let at = self.started_at.elapsed();
+ self.items += 1;
+ self.first_item.get_or_insert(at);
+ self.last_item = Some(at);
+ }
+ }
+
+ impl Drop for StreamLiveness {
+ // Recorded as `i64`: a `u64` reaches OpenTelemetry as a *string*
+ // attribute, which every numeric query would then silently miss.
+ fn drop(&mut self) {
+ self.span.record("agent.stream.items", self.items);
+ if let Some(first) = self.first_item {
+ self.span
+ .record("agent.stream.first_item_ms", millis(first));
+ }
+ // With nothing ever received the silence is the whole run, which is
+ // what `unwrap_or_default` says here.
+ let silent_since = self.last_item.unwrap_or_default();
+ self.span.record(
+ "agent.stream.trailing_silence_ms",
+ millis(self.started_at.elapsed().saturating_sub(silent_since)),
+ );
+ }
+ }
+
+ /// Whole milliseconds, saturating: a duration longer than `i64::MAX`
+ /// milliseconds is not a number anybody needs exactly.
+ fn millis(duration: Duration) -> i64 {
+ i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)
+ }
+
let mut finish_run = FinishRun {
telemetry: telemetry.clone(),
agent_span: agent_span.clone(),
@@ -509,20 +571,22 @@ where
let driver_span = agent_span.clone();
let driver = tokio::spawn(
async move {
- let mut thinking_buf = String::new();
+ let mut liveness = StreamLiveness::new(agent_span.clone());
while let Some(item) = rig_stream.next().await {
+ liveness.observed();
match item {
Ok(MultiTurnStreamItem::StreamAssistantItem(
StreamedAssistantContent::ReasoningDelta { reasoning, .. },
)) => {
- thinking_buf.push_str(&reasoning);
+ // Forwarded as it arrives, like a text delta. Held back
+ // until the model says something else, a turn that
+ // thinks for half a minute before its first tool call
+ // shows the reader nothing for that whole time and then
+ // the entire thought at once.
+ let _ = driver_tx.send(Ok(StreamPart::Thinking(reasoning)));
}
other => {
- if !thinking_buf.is_empty() {
- let _ = driver_tx
- .send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));
- }
match other {
Ok(MultiTurnStreamItem::FinalResponse(final_resp)) => {
let usage = final_resp.usage;
@@ -573,9 +637,6 @@ where
}
}
}
- if !thinking_buf.is_empty() {
- let _ = driver_tx.send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));
- }
// Dropping `rig_stream` (and with it the hook's sender) plus `driver_tx`
// here closes the channel, ending the consumer stream below.
drop(finish_run);
diff --git a/crates/agent/src/test/agent_loop/mod.rs b/crates/agent/src/test/agent_loop/mod.rs
index 6816d89245..6ae4f8b1a6 100644
--- a/crates/agent/src/test/agent_loop/mod.rs
+++ b/crates/agent/src/test/agent_loop/mod.rs
@@ -8,6 +8,7 @@ mod util;
mod test_cooperative_cancellation;
mod test_eager_tools;
mod test_telemetry;
+mod test_thinking;
mod test_tool;
mod test_tool_search;
mod test_user_tools;
diff --git a/crates/agent/src/test/agent_loop/test_telemetry.rs b/crates/agent/src/test/agent_loop/test_telemetry.rs
index f14a7f93ae..fe2ec68529 100644
--- a/crates/agent/src/test/agent_loop/test_telemetry.rs
+++ b/crates/agent/src/test/agent_loop/test_telemetry.rs
@@ -586,3 +586,56 @@ async fn a_loop_with_telemetry_off_records_no_genai_content() {
);
}
}
+
+/// Stream liveness: how many items the provider sent, when the first arrived,
+/// and how long it had been quiet when the run ended. Without these a parked
+/// stream is unreadable - a provider dribbling tokens and one that has gone
+/// silent look identical from the span's duration alone.
+#[tokio::test]
+async fn the_run_span_records_how_the_provider_stream_behaved() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+
+ let model = MockCompletionModel::from_stream_turns([vec![
+ MockStreamEvent::ReasoningDelta {
+ id: None,
+ reasoning: "thinking".to_owned(),
+ },
+ MockStreamEvent::Text("done".to_owned()),
+ MockStreamEvent::final_response_with_default_usage(),
+ ]]);
+
+ let mut session = util::test_loop()
+ .with_genai_telemetry(false)
+ .test_session(
+ util::tool_set(ai_toolset::AsyncToolCollection::<()>::new()),
+ Arc::new(()),
+ "test preamble",
+ util::usage_ctx(),
+ model,
+ )
+ .await;
+ util::drive(&mut session, "think then answer").await;
+ drop(session);
+
+ let spans = finished(&exporter, &provider);
+ let run = spans
+ .iter()
+ .find(|span| span.name == "agent.turn")
+ .unwrap_or_else(|| panic!("the run span, got {}", describe(&spans)));
+
+ // Numbers, not strings: a `u64` reaches OpenTelemetry as a string
+ // attribute and every numeric query on it silently misses.
+ assert!(
+ int_attribute(run, "agent.stream.items").is_some_and(|items| items >= 3),
+ "every item the driver saw is counted, as a number: {:?}",
+ attribute(run, "agent.stream.items")
+ );
+ assert!(
+ int_attribute(run, "agent.stream.first_item_ms").is_some(),
+ "the wait for the first item is recorded"
+ );
+ assert!(
+ int_attribute(run, "agent.stream.trailing_silence_ms").is_some(),
+ "the silence at the end is recorded"
+ );
+}
diff --git a/crates/agent/src/test/agent_loop/test_thinking.rs b/crates/agent/src/test/agent_loop/test_thinking.rs
new file mode 100644
index 0000000000..85e3d53ede
--- /dev/null
+++ b/crates/agent/src/test/agent_loop/test_thinking.rs
@@ -0,0 +1,92 @@
+//! Thinking reaches the consumer as it is produced.
+//!
+//! A reasoning model can think for tens of seconds before it says anything
+//! else. Holding those deltas until the next non-reasoning item - a tool call,
+//! text, the final response - shows the reader nothing for that whole stretch
+//! and then the entire thought at once, which is the opposite of what a
+//! thinking stream is for.
+
+use super::util;
+use crate::stream::StreamPart;
+use ai_toolset::AsyncToolCollection;
+use rig_core::message::Message;
+use rig_core::test_utils::{MockCompletionModel, MockStreamEvent};
+use std::sync::Arc;
+
+fn reasoning(text: &str) -> MockStreamEvent {
+ MockStreamEvent::ReasoningDelta {
+ id: None,
+ reasoning: text.to_owned(),
+ }
+}
+
+/// Each delta arrives as its own part, in order. One coalesced `Thinking`
+/// part would pass a "the thought came through" assertion just as well, which
+/// is why this pins the shape rather than the text.
+#[tokio::test]
+async fn reasoning_deltas_stream_as_they_arrive() {
+ let model = MockCompletionModel::from_stream_turns([vec![
+ reasoning("first "),
+ reasoning("second "),
+ reasoning("third"),
+ MockStreamEvent::final_response_with_default_usage(),
+ ]]);
+
+ let mut session = util::session(
+ util::tool_set(AsyncToolCollection::<()>::new()),
+ Arc::new(()),
+ model,
+ )
+ .await;
+ let collected = util::collect(
+ session
+ .send_message(vec![Message::user("think")])
+ .await
+ .expect("the stream starts"),
+ )
+ .await;
+
+ let thoughts: Vec<&str> = collected
+ .parts
+ .iter()
+ .filter_map(|part| match part {
+ StreamPart::Thinking(text) => Some(text.as_str()),
+ _ => None,
+ })
+ .collect();
+
+ assert_eq!(thoughts, vec!["first ", "second ", "third"]);
+ assert!(collected.error.is_none());
+}
+
+/// A turn that ends while still thinking - the model produced nothing else,
+/// or the run was cancelled - has still shown its reader everything it
+/// thought. Nothing waits on a final response that may never come.
+#[tokio::test]
+async fn thinking_reaches_the_consumer_without_a_following_item() {
+ let model =
+ MockCompletionModel::from_stream_turns([vec![reasoning("thought that stands alone")]]);
+
+ let mut session = util::session(
+ util::tool_set(AsyncToolCollection::<()>::new()),
+ Arc::new(()),
+ model,
+ )
+ .await;
+ let collected = util::collect(
+ session
+ .send_message(vec![Message::user("think")])
+ .await
+ .expect("the stream starts"),
+ )
+ .await;
+
+ assert!(
+ collected.parts.iter().any(|part| matches!(
+ part,
+ StreamPart::Thinking(text) if text == "thought that stands alone"
+ )),
+ "the thought reached the consumer: {:?}",
+ collected.parts
+ );
+}