Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(agent): record provider time to first chunk on agent-session turns (#7704)
Wolf Mermelsteinf2055ba64ea0
diff --git a/crates/agent/src/agent_loop.rs b/crates/agent/src/agent_loop.rsindex 78b90ce015..d73e1587bc 100644--- a/crates/agent/src/agent_loop.rs+++ b/crates/agent/src/agent_loop.rs@@ -559,7 +559,8 @@ impl Session {) -> Result<ChatCompletionStream<'_>, AgentError> {// `agent.stream.*` are recorded by the stream driver as the run ends;// see `StreamLiveness` in `crate::model::router`. Declared on both- // shapes of the span because either one can be the run's.+ // shapes of the span because either one can be the run's; `agent.turn`+ // names the model itself, `invoke_agent` through `gen_ai.request.model`.let span = if self.telemetry.enabled() {tracing::info_span!("invoke_agent",@@ -569,15 +570,18 @@ impl Session {gen_ai.provider.name = self.telemetry.provider_name(),gen_ai.request.model = self.telemetry.model_name(),agent.stream.items = tracing::field::Empty,- agent.stream.first_item_ms = tracing::field::Empty,+ agent.stream.first_chunk_ms = tracing::field::Empty,+ agent.stream.first_chunk_kind = tracing::field::Empty,agent.stream.trailing_silence_ms = tracing::field::Empty,)} else {tracing::info_span!("agent.turn",agent.name = %self.telemetry.agent_name(),+ agent.model = self.telemetry.model_name(),agent.stream.items = tracing::field::Empty,- agent.stream.first_item_ms = tracing::field::Empty,+ agent.stream.first_chunk_ms = tracing::field::Empty,+ agent.stream.first_chunk_kind = tracing::field::Empty,agent.stream.trailing_silence_ms = tracing::field::Empty,)};diff --git a/crates/agent/src/model/router.rs b/crates/agent/src/model/router.rsindex 4d7bf245e0..51e63443cc 100644--- a/crates/agent/src/model/router.rs+++ b/crates/agent/src/model/router.rs@@ -672,30 +672,34 @@ where/// the items tells those apart. Read `trailing_silence_ms` first - near/// zero means the model was still producing when the run ended, a long tail/// means it had stopped talking to us well before.+ ///+ /// `first_chunk_ms` is the run's first model call's time to first chunk,+ /// measured from the provider request (the same value as that call's+ /// `chat` span's `macro.genai.chat.time_to_first_chunk_ms`), so the+ /// provider's latency reads off the run span without a join.struct StreamLiveness {span: tracing::Span,+ telemetry: GenAiContext,started_at: Instant,items: i64,- first_item: Option<Duration>,last_item: Option<Duration>,}impl StreamLiveness {- fn new(span: tracing::Span) -> Self {+ fn new(span: tracing::Span, telemetry: GenAiContext) -> Self {+ telemetry.take_run_first_chunk();Self {span,+ telemetry,started_at: Instant::now(),items: 0,- first_item: None,last_item: None,}}fn observed(&mut self) {- let at = self.started_at.elapsed();self.items += 1;- self.first_item.get_or_insert(at);- self.last_item = Some(at);+ self.last_item = Some(self.started_at.elapsed());}}@@ -704,9 +708,10 @@ where// attribute, which every numeric query would then silently miss.fn drop(&mut self) {self.span.record("agent.stream.items", self.items);- if let Some(first) = self.first_item {+ if let Some((time_to_first_chunk, kind)) = self.telemetry.take_run_first_chunk() {self.span- .record("agent.stream.first_item_ms", millis(first));+ .record("agent.stream.first_chunk_ms", millis(time_to_first_chunk));+ self.span.record("agent.stream.first_chunk_kind", kind);}// With nothing ever received the silence is the whole run, which is// what `unwrap_or_default` says here.@@ -733,7 +738,7 @@ wherelet financial_context = MeteringContext::current();let driver = tokio::spawn(MeteringContext::carry(financial_context, async move {- let mut liveness = StreamLiveness::new(agent_span.clone());+ let mut liveness = StreamLiveness::new(agent_span.clone(), telemetry.clone());while let Some(item) = rig_stream.next().await {liveness.observed();diff --git a/crates/agent/src/telemetry.rs b/crates/agent/src/telemetry.rsindex 4c9fe9a502..f75a29623e 100644--- a/crates/agent/src/telemetry.rs+++ b/crates/agent/src/telemetry.rs@@ -27,7 +27,7 @@mod test;use std::sync::{Arc, Mutex, OnceLock};-use std::time::Instant;+use std::time::{Duration, Instant};use genai_telemetry::messages::{self, MediaSource};use genai_telemetry::{@@ -56,8 +56,8 @@ struct Inner {/// Label for the agent, e.g. the AI feature driving it.agent_name: String,policy: ContentPolicy,- /// Whether anything is recorded at all. Off, every `record_*` is a no-op- /// and the model call's span is never parked.+ /// Whether GenAI content and usage are recorded. Off, only each model+ /// call's first-chunk timing is: nothing else measures it.enabled: bool,/// The routed `(provider, model)`, set once routing has decided.model: OnceLock<(String, String)>,@@ -68,6 +68,9 @@ struct Inner {/// When the parked model call was requested, until its first chunk is/// recorded.chat_requested_at: Mutex<Option<Instant>>,+ /// The time to first chunk and its kind of the current run's first model+ /// call, until [`Self::take_run_first_chunk`].+ run_first_chunk: Mutex<Option<(Duration, &'static str)>>,}impl GenAiContext {@@ -85,6 +88,7 @@ impl GenAiContext {model: OnceLock::new(),chat_span: Mutex::new(None),chat_requested_at: Mutex::new(None),+ run_first_chunk: Mutex::new(None),}))}@@ -119,10 +123,19 @@ impl GenAiContext {/// `chat` span when called from [`TracedModel`]) and park that span for/// [`Self::record_response`].fn record_request(&self, request: &CompletionRequest) {- if !self.0.enabled {- return;- }let span = tracing::Span::current();+ if self.0.enabled {+ self.record_request_fields(&span, request);+ }+ *self.0.chat_span.lock().expect("chat span slot poisoned") = Some(span);+ *self+ .0+ .chat_requested_at+ .lock()+ .expect("chat request time poisoned") = Some(Instant::now());+ }++ fn record_request_fields(&self, span: &tracing::Span, request: &CompletionRequest) {let policy = &self.0.policy;if let Some(conversation_id) = self.conversation_id() {@@ -161,13 +174,6 @@ impl GenAiContext {span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);}}-- *self.0.chat_span.lock().expect("chat span slot poisoned") = Some(span);- *self- .0- .chat_requested_at- .lock()- .expect("chat request time poisoned") = Some(Instant::now());}/// The model call in flight streamed a chunk of `kind`. The first one per@@ -182,6 +188,12 @@ impl GenAiContext {else {return;};+ let time_to_first_chunk = requested_at.elapsed();+ self.0+ .run_first_chunk+ .lock()+ .expect("run first chunk poisoned")+ .get_or_insert((time_to_first_chunk, kind));if let Some(span) = self.0.chat_span@@ -191,12 +203,23 @@ impl GenAiContext {{span.set_i64(attr::MACRO_CHAT_TIME_TO_FIRST_CHUNK_MS,- i64::try_from(requested_at.elapsed().as_millis()).unwrap_or(i64::MAX),+ i64::try_from(time_to_first_chunk.as_millis()).unwrap_or(i64::MAX),);span.set_str(attr::MACRO_CHAT_FIRST_CHUNK_KIND, kind);}}+ /// The time to first chunk and its kind of the first model call recorded+ /// since the last take: taken as a run starts, to clear the slot, and as+ /// it ends.+ pub(crate) fn take_run_first_chunk(&self) -> Option<(Duration, &'static str)> {+ self.0+ .run_first_chunk+ .lock()+ .expect("run first chunk poisoned")+ .take()+ }+/// Record the response side of the model call whose request was recorded/// last, then release the parked span so it can close.fn record_response(&self, content: &OneOrMany<AssistantContent>) {@@ -208,6 +231,10 @@ impl GenAiContext {AssistantContent::Reasoning(_) => "reasoning",AssistantContent::Image(_) => "image",});+ if !self.0.enabled {+ self.finish_run();+ return;+ }let Some(span) = self.0.chat_span@@ -250,6 +277,7 @@ impl GenAiContext {detail: &str,) {if !self.0.enabled {+ self.finish_run();return;}let description = self.failure_description(error_type, detail);@@ -264,6 +292,7 @@ impl GenAiContext {/// on every path - a one-shot completion has no run driver to do it.fn record_chat_failure(&self, error: &CompletionError) {if !self.0.enabled {+ self.finish_run();return;}let description = self.failure_description("provider_error", &error.to_string());diff --git a/crates/agent/src/test/agent_loop/test_telemetry.rs b/crates/agent/src/test/agent_loop/test_telemetry.rsindex 99447d2253..ac5db12820 100644--- a/crates/agent/src/test/agent_loop/test_telemetry.rs+++ b/crates/agent/src/test/agent_loop/test_telemetry.rs@@ -587,8 +587,9 @@ async fn a_loop_with_telemetry_off_records_no_genai_content() {}}-/// Stream liveness: how many items the provider sent, when the first arrived,-/// and how long it had been quiet when the run ended. Without these a parked+/// Stream liveness: how many items the provider sent, how soon the first+/// model call's first chunk came and from which model, and how long it had+/// been quiet when the run ended. Without these a parked/// stream is unreadable - a provider dribbling tokens and one that has gone/// silent look identical from the span's duration alone.#[tokio::test]@@ -631,8 +632,17 @@ async fn the_run_span_records_how_the_provider_stream_behaved() {attribute(run, "agent.stream.items"));assert!(- int_attribute(run, "agent.stream.first_item_ms").is_some(),- "the wait for the first item is recorded"+ int_attribute(run, "agent.stream.first_chunk_ms").is_some_and(|ms| ms >= 0),+ "the provider's time to first chunk is recorded, as a number: {:?}",+ attribute(run, "agent.stream.first_chunk_ms")+ );+ assert_eq!(+ string_attribute(run, "agent.stream.first_chunk_kind").as_deref(),+ Some("reasoning")+ );+ assert_eq!(+ string_attribute(run, "agent.model").as_deref(),+ Some("fake-model"));assert!(int_attribute(run, "agent.stream.trailing_silence_ms").is_some(),@@ -687,3 +697,79 @@ async fn each_model_call_records_its_first_chunk() {);}}++/// The first-chunk timing is the one thing recorded with GenAI telemetry off:+/// an agent session's turns are reported by the session actor, which cannot+/// see the provider request, so nothing else measures it.+#[tokio::test]+async fn a_loop_with_telemetry_off_still_records_each_first_chunk() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let model = MockCompletionModel::from_stream_turns([+ vec![+ MockStreamEvent::tool_call("call-1", "echo_tool", json!({ "value": "a" })),+ MockStreamEvent::final_response_with_default_usage(),+ ],+ vec![+ MockStreamEvent::text("done"),+ MockStreamEvent::final_response_with_default_usage(),+ ],+ ]);+ let toolset = util::single_tool_set::<EchoTool, ()>();+ let mut session = util::test_loop()+ .with_genai_telemetry(false)+ .test_session(+ toolset,+ Arc::new(()),+ "test preamble",+ util::usage_ctx(),+ model,+ )+ .await;++ let result = util::drive(&mut session, "call echo").await;+ assert_eq!(result.content(), "done");+ drop(session);++ let spans = finished(&exporter, &provider);+ let chats = spans_with_operation(&spans, attr::operation::CHAT);+ assert_eq!(+ chats.len(),+ 2,+ "two model calls, got:\n{}",+ describe(&spans)+ );+ assert_eq!(+ string_attribute(chats[0], attr::MACRO_CHAT_FIRST_CHUNK_KIND).as_deref(),+ Some("tool_call")+ );+ assert_eq!(+ string_attribute(chats[1], attr::MACRO_CHAT_FIRST_CHUNK_KIND).as_deref(),+ Some("text")+ );+ for chat in &chats {+ assert!(+ int_attribute(chat, attr::MACRO_CHAT_TIME_TO_FIRST_CHUNK_MS).is_some_and(|ms| ms >= 0),+ "{}",+ describe(&spans)+ );+ assert_eq!(+ string_array_attribute(chat, attr::RESPONSE_FINISH_REASONS),+ None,+ "the response side stays unrecorded: {chat:#?}"+ );+ }++ let run = spans+ .iter()+ .find(|span| span.name == "agent.turn")+ .unwrap_or_else(|| panic!("the run span, got {}", describe(&spans)));+ assert_eq!(+ int_attribute(run, "agent.stream.first_chunk_ms"),+ int_attribute(chats[0], attr::MACRO_CHAT_TIME_TO_FIRST_CHUNK_MS),+ "the run span carries its first model call's time to first chunk"+ );+ assert_eq!(+ string_attribute(run, "agent.stream.first_chunk_kind").as_deref(),+ Some("tool_call")+ );+}