Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(agent): record provider time to first chunk on agent-session turns (#7704)

Wolf Mermelsteinf2055ba64ea0
diff --git a/crates/agent/src/agent_loop.rs b/crates/agent/src/agent_loop.rs
index 78b90ce015..d73e1587bc 100644
--- a/crates/agent/src/agent_loop.rs
+++ b/crates/agent/src/agent_loop.rs
@@ -559,7 +559,8 @@ impl Session {
) -> Result<ChatCompletionStream<'_>, AgentError> {
// `agent.stream.*` are recorded by the stream driver as the run ends;
// see `StreamLiveness` in `crate::model::router`. Declared on both
- // shapes of the span because either one can be the run's.
+ // shapes of the span because either one can be the run's; `agent.turn`
+ // names the model itself, `invoke_agent` through `gen_ai.request.model`.
let span = if self.telemetry.enabled() {
tracing::info_span!(
"invoke_agent",
@@ -569,15 +570,18 @@ impl Session {
gen_ai.provider.name = self.telemetry.provider_name(),
gen_ai.request.model = self.telemetry.model_name(),
agent.stream.items = tracing::field::Empty,
- agent.stream.first_item_ms = tracing::field::Empty,
+ agent.stream.first_chunk_ms = tracing::field::Empty,
+ agent.stream.first_chunk_kind = tracing::field::Empty,
agent.stream.trailing_silence_ms = tracing::field::Empty,
)
} else {
tracing::info_span!(
"agent.turn",
agent.name = %self.telemetry.agent_name(),
+ agent.model = self.telemetry.model_name(),
agent.stream.items = tracing::field::Empty,
- agent.stream.first_item_ms = tracing::field::Empty,
+ agent.stream.first_chunk_ms = tracing::field::Empty,
+ agent.stream.first_chunk_kind = tracing::field::Empty,
agent.stream.trailing_silence_ms = tracing::field::Empty,
)
};
diff --git a/crates/agent/src/model/router.rs b/crates/agent/src/model/router.rs
index 4d7bf245e0..51e63443cc 100644
--- a/crates/agent/src/model/router.rs
+++ b/crates/agent/src/model/router.rs
@@ -672,30 +672,34 @@ where
/// the items tells those apart. Read `trailing_silence_ms` first - near
/// zero means the model was still producing when the run ended, a long tail
/// means it had stopped talking to us well before.
+ ///
+ /// `first_chunk_ms` is the run's first model call's time to first chunk,
+ /// measured from the provider request (the same value as that call's
+ /// `chat` span's `macro.genai.chat.time_to_first_chunk_ms`), so the
+ /// provider's latency reads off the run span without a join.
struct StreamLiveness {
span: tracing::Span,
+ telemetry: GenAiContext,
started_at: Instant,
items: i64,
- first_item: Option<Duration>,
last_item: Option<Duration>,
}
impl StreamLiveness {
- fn new(span: tracing::Span) -> Self {
+ fn new(span: tracing::Span, telemetry: GenAiContext) -> Self {
+ telemetry.take_run_first_chunk();
Self {
span,
+ telemetry,
started_at: Instant::now(),
items: 0,
- first_item: None,
last_item: None,
}
}
fn observed(&mut self) {
- let at = self.started_at.elapsed();
self.items += 1;
- self.first_item.get_or_insert(at);
- self.last_item = Some(at);
+ self.last_item = Some(self.started_at.elapsed());
}
}
@@ -704,9 +708,10 @@ where
// attribute, which every numeric query would then silently miss.
fn drop(&mut self) {
self.span.record("agent.stream.items", self.items);
- if let Some(first) = self.first_item {
+ if let Some((time_to_first_chunk, kind)) = self.telemetry.take_run_first_chunk() {
self.span
- .record("agent.stream.first_item_ms", millis(first));
+ .record("agent.stream.first_chunk_ms", millis(time_to_first_chunk));
+ self.span.record("agent.stream.first_chunk_kind", kind);
}
// With nothing ever received the silence is the whole run, which is
// what `unwrap_or_default` says here.
@@ -733,7 +738,7 @@ where
let financial_context = MeteringContext::current();
let driver = tokio::spawn(
MeteringContext::carry(financial_context, async move {
- let mut liveness = StreamLiveness::new(agent_span.clone());
+ let mut liveness = StreamLiveness::new(agent_span.clone(), telemetry.clone());
while let Some(item) = rig_stream.next().await {
liveness.observed();
diff --git a/crates/agent/src/telemetry.rs b/crates/agent/src/telemetry.rs
index 4c9fe9a502..f75a29623e 100644
--- a/crates/agent/src/telemetry.rs
+++ b/crates/agent/src/telemetry.rs
@@ -27,7 +27,7 @@
mod test;
use std::sync::{Arc, Mutex, OnceLock};
-use std::time::Instant;
+use std::time::{Duration, Instant};
use genai_telemetry::messages::{self, MediaSource};
use genai_telemetry::{
@@ -56,8 +56,8 @@ struct Inner {
/// Label for the agent, e.g. the AI feature driving it.
agent_name: String,
policy: ContentPolicy,
- /// Whether anything is recorded at all. Off, every `record_*` is a no-op
- /// and the model call's span is never parked.
+ /// Whether GenAI content and usage are recorded. Off, only each model
+ /// call's first-chunk timing is: nothing else measures it.
enabled: bool,
/// The routed `(provider, model)`, set once routing has decided.
model: OnceLock<(String, String)>,
@@ -68,6 +68,9 @@ struct Inner {
/// When the parked model call was requested, until its first chunk is
/// recorded.
chat_requested_at: Mutex<Option<Instant>>,
+ /// The time to first chunk and its kind of the current run's first model
+ /// call, until [`Self::take_run_first_chunk`].
+ run_first_chunk: Mutex<Option<(Duration, &'static str)>>,
}
impl GenAiContext {
@@ -85,6 +88,7 @@ impl GenAiContext {
model: OnceLock::new(),
chat_span: Mutex::new(None),
chat_requested_at: Mutex::new(None),
+ run_first_chunk: Mutex::new(None),
}))
}
@@ -119,10 +123,19 @@ impl GenAiContext {
/// `chat` span when called from [`TracedModel`]) and park that span for
/// [`Self::record_response`].
fn record_request(&self, request: &CompletionRequest) {
- if !self.0.enabled {
- return;
- }
let span = tracing::Span::current();
+ if self.0.enabled {
+ self.record_request_fields(&span, request);
+ }
+ *self.0.chat_span.lock().expect("chat span slot poisoned") = Some(span);
+ *self
+ .0
+ .chat_requested_at
+ .lock()
+ .expect("chat request time poisoned") = Some(Instant::now());
+ }
+
+ fn record_request_fields(&self, span: &tracing::Span, request: &CompletionRequest) {
let policy = &self.0.policy;
if let Some(conversation_id) = self.conversation_id() {
@@ -161,13 +174,6 @@ impl GenAiContext {
span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
}
}
-
- *self.0.chat_span.lock().expect("chat span slot poisoned") = Some(span);
- *self
- .0
- .chat_requested_at
- .lock()
- .expect("chat request time poisoned") = Some(Instant::now());
}
/// The model call in flight streamed a chunk of `kind`. The first one per
@@ -182,6 +188,12 @@ impl GenAiContext {
else {
return;
};
+ let time_to_first_chunk = requested_at.elapsed();
+ self.0
+ .run_first_chunk
+ .lock()
+ .expect("run first chunk poisoned")
+ .get_or_insert((time_to_first_chunk, kind));
if let Some(span) = self
.0
.chat_span
@@ -191,12 +203,23 @@ impl GenAiContext {
{
span.set_i64(
attr::MACRO_CHAT_TIME_TO_FIRST_CHUNK_MS,
- i64::try_from(requested_at.elapsed().as_millis()).unwrap_or(i64::MAX),
+ i64::try_from(time_to_first_chunk.as_millis()).unwrap_or(i64::MAX),
);
span.set_str(attr::MACRO_CHAT_FIRST_CHUNK_KIND, kind);
}
}
+ /// The time to first chunk and its kind of the first model call recorded
+ /// since the last take: taken as a run starts, to clear the slot, and as
+ /// it ends.
+ pub(crate) fn take_run_first_chunk(&self) -> Option<(Duration, &'static str)> {
+ self.0
+ .run_first_chunk
+ .lock()
+ .expect("run first chunk poisoned")
+ .take()
+ }
+
/// Record the response side of the model call whose request was recorded
/// last, then release the parked span so it can close.
fn record_response(&self, content: &OneOrMany<AssistantContent>) {
@@ -208,6 +231,10 @@ impl GenAiContext {
AssistantContent::Reasoning(_) => "reasoning",
AssistantContent::Image(_) => "image",
});
+ if !self.0.enabled {
+ self.finish_run();
+ return;
+ }
let Some(span) = self
.0
.chat_span
@@ -250,6 +277,7 @@ impl GenAiContext {
detail: &str,
) {
if !self.0.enabled {
+ self.finish_run();
return;
}
let description = self.failure_description(error_type, detail);
@@ -264,6 +292,7 @@ impl GenAiContext {
/// on every path - a one-shot completion has no run driver to do it.
fn record_chat_failure(&self, error: &CompletionError) {
if !self.0.enabled {
+ self.finish_run();
return;
}
let description = self.failure_description("provider_error", &error.to_string());
diff --git a/crates/agent/src/test/agent_loop/test_telemetry.rs b/crates/agent/src/test/agent_loop/test_telemetry.rs
index 99447d2253..ac5db12820 100644
--- a/crates/agent/src/test/agent_loop/test_telemetry.rs
+++ b/crates/agent/src/test/agent_loop/test_telemetry.rs
@@ -587,8 +587,9 @@ async fn a_loop_with_telemetry_off_records_no_genai_content() {
}
}
-/// Stream liveness: how many items the provider sent, when the first arrived,
-/// and how long it had been quiet when the run ended. Without these a parked
+/// Stream liveness: how many items the provider sent, how soon the first
+/// model call's first chunk came and from which model, and how long it had
+/// been quiet when the run ended. Without these a parked
/// stream is unreadable - a provider dribbling tokens and one that has gone
/// silent look identical from the span's duration alone.
#[tokio::test]
@@ -631,8 +632,17 @@ async fn the_run_span_records_how_the_provider_stream_behaved() {
attribute(run, "agent.stream.items")
);
assert!(
- int_attribute(run, "agent.stream.first_item_ms").is_some(),
- "the wait for the first item is recorded"
+ int_attribute(run, "agent.stream.first_chunk_ms").is_some_and(|ms| ms >= 0),
+ "the provider's time to first chunk is recorded, as a number: {:?}",
+ attribute(run, "agent.stream.first_chunk_ms")
+ );
+ assert_eq!(
+ string_attribute(run, "agent.stream.first_chunk_kind").as_deref(),
+ Some("reasoning")
+ );
+ assert_eq!(
+ string_attribute(run, "agent.model").as_deref(),
+ Some("fake-model")
);
assert!(
int_attribute(run, "agent.stream.trailing_silence_ms").is_some(),
@@ -687,3 +697,79 @@ async fn each_model_call_records_its_first_chunk() {
);
}
}
+
+/// The first-chunk timing is the one thing recorded with GenAI telemetry off:
+/// an agent session's turns are reported by the session actor, which cannot
+/// see the provider request, so nothing else measures it.
+#[tokio::test]
+async fn a_loop_with_telemetry_off_still_records_each_first_chunk() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let model = MockCompletionModel::from_stream_turns([
+ vec![
+ MockStreamEvent::tool_call("call-1", "echo_tool", json!({ "value": "a" })),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::text("done"),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ ]);
+ let toolset = util::single_tool_set::<EchoTool, ()>();
+ let mut session = util::test_loop()
+ .with_genai_telemetry(false)
+ .test_session(
+ toolset,
+ Arc::new(()),
+ "test preamble",
+ util::usage_ctx(),
+ model,
+ )
+ .await;
+
+ let result = util::drive(&mut session, "call echo").await;
+ assert_eq!(result.content(), "done");
+ drop(session);
+
+ let spans = finished(&exporter, &provider);
+ let chats = spans_with_operation(&spans, attr::operation::CHAT);
+ assert_eq!(
+ chats.len(),
+ 2,
+ "two model calls, got:\n{}",
+ describe(&spans)
+ );
+ assert_eq!(
+ string_attribute(chats[0], attr::MACRO_CHAT_FIRST_CHUNK_KIND).as_deref(),
+ Some("tool_call")
+ );
+ assert_eq!(
+ string_attribute(chats[1], attr::MACRO_CHAT_FIRST_CHUNK_KIND).as_deref(),
+ Some("text")
+ );
+ for chat in &chats {
+ assert!(
+ int_attribute(chat, attr::MACRO_CHAT_TIME_TO_FIRST_CHUNK_MS).is_some_and(|ms| ms >= 0),
+ "{}",
+ describe(&spans)
+ );
+ assert_eq!(
+ string_array_attribute(chat, attr::RESPONSE_FINISH_REASONS),
+ None,
+ "the response side stays unrecorded: {chat:#?}"
+ );
+ }
+
+ let run = spans
+ .iter()
+ .find(|span| span.name == "agent.turn")
+ .unwrap_or_else(|| panic!("the run span, got {}", describe(&spans)));
+ assert_eq!(
+ int_attribute(run, "agent.stream.first_chunk_ms"),
+ int_attribute(chats[0], attr::MACRO_CHAT_TIME_TO_FIRST_CHUNK_MS),
+ "the run span carries its first model call's time to first chunk"
+ );
+ assert_eq!(
+ string_attribute(run, "agent.stream.first_chunk_kind").as_deref(),
+ Some("tool_call")
+ );
+}