Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(observability): warm-claim miss reasons and handshake wait (#7748)
Wolf Mermelsteind3e3eda9e094
diff --git a/.sqlx/query-1e2b5f16fc1eca3b9f8261470249db35499a25f4251d8b62d7ae5a91ad7b7182.json b/.sqlx/query-1e2b5f16fc1eca3b9f8261470249db35499a25f4251d8b62d7ae5a91ad7b7182.jsonnew file mode 100644index 0000000000..a898f659bb--- /dev/null+++ b/.sqlx/query-1e2b5f16fc1eca3b9f8261470249db35499a25f4251d8b62d7ae5a91ad7b7182.json@@ -0,0 +1,74 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH candidate AS (\n SELECT\n true AS found,\n list_hidden AND thread_id IS NULL AS warm,\n owner_id = $2 AS owner_matches,\n bot_id = $3 AS bot_matches,\n model = $4 AS model_matches,\n instructions IS NOT DISTINCT FROM $5 AS instructions_match,\n COALESCE(status != 'disconnected'\n AND NOT (status = 'event' AND status_event_name = 'disconnected'), false) AS live,\n created_at > NOW() - INTERVAL '10 minutes' AS fresh\n FROM agent_session WHERE id = $1\n ), claimed AS (\n UPDATE agent_session SET list_hidden = false\n WHERE id = $1 AND owner_id = $2 AND bot_id = $3 AND model = $4\n AND instructions IS NOT DISTINCT FROM $5\n AND list_hidden AND thread_id IS NULL\n AND status != 'disconnected'\n AND NOT (status = 'event' AND status_event_name = 'disconnected')\n AND created_at > NOW() - INTERVAL '10 minutes'\n RETURNING id\n )\n SELECT\n EXISTS (SELECT 1 FROM claimed) AS \"claimed!\",\n candidate.found AS \"found?\",\n candidate.warm AS \"warm?\",\n candidate.owner_matches AS \"owner_matches?\",\n candidate.bot_matches AS \"bot_matches?\",\n candidate.model_matches AS \"model_matches?\",\n candidate.instructions_match AS \"instructions_match?\",\n candidate.live AS \"live?\",\n candidate.fresh AS \"fresh?\"\n FROM (SELECT 1) AS one LEFT JOIN candidate ON true\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "claimed!",+ "type_info": "Bool"+ },+ {+ "ordinal": 1,+ "name": "found?",+ "type_info": "Bool"+ },+ {+ "ordinal": 2,+ "name": "warm?",+ "type_info": "Bool"+ },+ {+ "ordinal": 3,+ "name": "owner_matches?",+ "type_info": "Bool"+ },+ {+ "ordinal": 4,+ "name": "bot_matches?",+ "type_info": "Bool"+ },+ {+ "ordinal": 5,+ "name": "model_matches?",+ "type_info": "Bool"+ },+ {+ "ordinal": 6,+ "name": "instructions_match?",+ "type_info": "Bool"+ },+ {+ "ordinal": 7,+ "name": "live?",+ "type_info": "Bool"+ },+ {+ "ordinal": 8,+ "name": "fresh?",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Text",+ "Uuid",+ "Text",+ "Text"+ ]+ },+ "nullable": [+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null+ ]+ },+ "hash": "1e2b5f16fc1eca3b9f8261470249db35499a25f4251d8b62d7ae5a91ad7b7182"+}diff --git a/.sqlx/query-7b29542761b4999a6e3525de2a86047f8612c24d28aeb03e777c024dd9a95938.json b/.sqlx/query-7b29542761b4999a6e3525de2a86047f8612c24d28aeb03e777c024dd9a95938.jsondeleted file mode 100644index eb76391d9a..0000000000--- a/.sqlx/query-7b29542761b4999a6e3525de2a86047f8612c24d28aeb03e777c024dd9a95938.json+++ /dev/null@@ -1,18 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n UPDATE agent_session SET list_hidden = false\n WHERE id = $1 AND owner_id = $2 AND bot_id = $3 AND model = $4\n AND instructions IS NOT DISTINCT FROM $5\n AND list_hidden AND thread_id IS NULL\n AND status != 'disconnected'\n AND NOT (status = 'event' AND status_event_name = 'disconnected')\n AND created_at > NOW() - INTERVAL '10 minutes'\n ",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Uuid",- "Text",- "Uuid",- "Text",- "Text"- ]- },- "nullable": []- },- "hash": "7b29542761b4999a6e3525de2a86047f8612c24d28aeb03e777c024dd9a95938"-}diff --git a/Cargo.lock b/Cargo.lockindex fab57d3179..c3f5e41c94 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -373,6 +373,7 @@ dependencies = ["model_user","notification","opentelemetry",+ "opentelemetry_sdk","pipedream_mcp","redis","reqwest 0.13.4",@@ -389,6 +390,7 @@ dependencies = ["tower 0.5.3","tracing","tracing-opentelemetry",+ "tracing-subscriber","url","utoipa","uuid",diff --git a/crates/agent_harness/Cargo.toml b/crates/agent_harness/Cargo.tomlindex ebee8170cf..8cd1213c4e 100644--- a/crates/agent_harness/Cargo.toml+++ b/crates/agent_harness/Cargo.toml@@ -91,5 +91,7 @@ bot_id = { path = "../bot_id", features = ["test-utils"] }harness_id = { path = "../harness_id", features = ["test-utils"] }mockall = { workspace = true }model_user = { path = "../model_user" }+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }tower = { workspace = true }+tracing-subscriber = { workspace = true }wiremock = { workspace = true }diff --git a/crates/agent_harness/src/domain/service/test/warm.rs b/crates/agent_harness/src/domain/service/test/warm.rsindex 0c028541f2..1f70fd287c 100644--- a/crates/agent_harness/src/domain/service/test/warm.rs+++ b/crates/agent_harness/src/domain/service/test/warm.rs@@ -1,7 +1,8 @@use super::*;+use agent_session::domain::warm::{WarmClaim, WarmMissReason};struct WarmLifecycleStub {- claim: bool,+ claim: WarmClaim,}impl agent_session::domain::warm::WarmSessionLifecycle for WarmLifecycleStub {fn claim<'a>(@@ -13,7 +14,7 @@ impl agent_session::domain::warm::WarmSessionLifecycle for WarmLifecycleStub {_instructions: Option<&'a str>,) -> std::pin::Pin<Box<- dyn std::future::Future<Output = agent_session::domain::error::Result<bool>>+ dyn std::future::Future<Output = agent_session::domain::error::Result<WarmClaim>>+ Send+ 'a,>,@@ -38,7 +39,9 @@ impl agent_session::domain::warm::WarmSessionLifecycle for WarmLifecycleStub {async fn warming_rejects_bot_owners_and_bounds_speculation_per_user() {let (service, _, containers, _, _) = harness();let service = service.with_warm_sessions(- Arc::new(WarmLifecycleStub { claim: false }),+ Arc::new(WarmLifecycleStub {+ claim: WarmClaim::Missed(WarmMissReason::NotFound),+ }),Arc::new(agent_session::domain::ports::NoOpToolCatalog),);assert!(matches!(@@ -95,8 +98,12 @@ async fn warming_initializes_without_a_prompt_or_visible_created_event() {let ((service, repo, containers, _, _), signals) =harness_with_signals(PromptContextMock::default(), PromptComposerMock::default());let tools = Arc::new(GatedWarmTools(tokio::sync::Semaphore::new(0)));- let service =- service.with_warm_sessions(Arc::new(WarmLifecycleStub { claim: true }), tools.clone());+ let service = service.with_warm_sessions(+ Arc::new(WarmLifecycleStub {+ claim: WarmClaim::Claimed,+ }),+ tools.clone(),+ );let id = AgentSessionId::new();let mut warm = Box::pin(service.warm_session(model_owner::Owner::User(sender()), id));let drive = async {@@ -152,3 +159,172 @@ async fn warming_initializes_without_a_prompt_or_visible_created_event() {"claim reveals the conversation");}++fn span_exporter() -> (+ opentelemetry_sdk::trace::InMemorySpanExporter,+ opentelemetry_sdk::trace::SdkTracerProvider,+ tracing::subscriber::DefaultGuard,+) {+ use opentelemetry::trace::TracerProvider as _;+ use tracing_subscriber::layer::SubscriberExt as _;+ let exporter = opentelemetry_sdk::trace::InMemorySpanExporter::default();+ let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ tracing::callsite::rebuild_interest_cache();+ (exporter, provider, guard)+}++fn attributes(+ exporter: &opentelemetry_sdk::trace::InMemorySpanExporter,+ provider: &opentelemetry_sdk::trace::SdkTracerProvider,+ name: &str,+) -> std::collections::HashMap<String, opentelemetry::Value> {+ provider.force_flush().expect("flush");+ let spans = exporter.get_finished_spans().expect("finished spans");+ let span = spans+ .iter()+ .find(|span| span.name == name)+ .unwrap_or_else(|| {+ panic!(+ "no {name} span in {:?}",+ spans.iter().map(|span| &span.name).collect::<Vec<_>>()+ )+ });+ span.attributes+ .iter()+ .map(|kv| (kv.key.to_string(), kv.value.clone()))+ .collect()+}++#[tokio::test]+async fn a_missed_warm_claim_records_why_on_the_open_span() {+ let (exporter, provider, _guard) = span_exporter();+ let (service, _, _, _, _) = harness();+ let service = service.with_warm_sessions(+ Arc::new(WarmLifecycleStub {+ claim: WarmClaim::Missed(WarmMissReason::ModelMismatch),+ }),+ Arc::new(agent_session::domain::ports::NoOpToolCatalog),+ );+ let session = service+ .open_managed_session(OpenManagedSession {+ id: Some(AgentSessionId::new()),+ owner: model_owner::Owner::User(sender()),+ prompt: None,+ repo_url: None,+ repo_branch: None,+ instructions: None,+ model: Some("composer-model".to_owned()),+ profile: Some(agent_session::domain::ports::SelectedManagedPersona {+ bot_id: bot_id::MACRO_NEW_BOT_ID,+ profile: None,+ }),+ })+ .await+ .expect("cold open after a missed claim");+ // The session's actor holds the open span until the session ends.+ service+ .execute(session.id, HarnessCommand::Delete)+ .await+ .expect("delete");++ let attributes = attributes(&exporter, &provider, "open_managed_session_inner");+ assert_eq!(+ attributes.get("agent.session.warm_miss_reason"),+ Some(&opentelemetry::Value::from("model_mismatch"))+ );+ assert_eq!(+ attributes.get("agent.session.warm_id_sent"),+ Some(&opentelemetry::Value::Bool(true))+ );+ assert_eq!(+ attributes.get("agent.session.warm_hit"),+ Some(&opentelemetry::Value::Bool(false))+ );+ assert_eq!(+ attributes.get("agent.session.model"),+ Some(&opentelemetry::Value::from("composer-model"))+ );+}++#[tokio::test]+async fn an_open_without_a_client_id_records_that_no_warm_id_was_sent() {+ let (exporter, provider, _guard) = span_exporter();+ let (service, _, _, _, _) = harness();+ let service = service.with_warm_sessions(+ Arc::new(WarmLifecycleStub {+ claim: WarmClaim::Missed(WarmMissReason::NotFound),+ }),+ Arc::new(agent_session::domain::ports::NoOpToolCatalog),+ );+ let session = service+ .open_managed_session(OpenManagedSession {+ id: None,+ owner: model_owner::Owner::User(sender()),+ prompt: None,+ repo_url: None,+ repo_branch: None,+ instructions: None,+ model: None,+ profile: Some(agent_session::domain::ports::SelectedManagedPersona {+ bot_id: bot_id::MACRO_NEW_BOT_ID,+ profile: None,+ }),+ })+ .await+ .expect("cold open");+ // The session's actor holds the open span until the session ends.+ service+ .execute(session.id, HarnessCommand::Delete)+ .await+ .expect("delete");++ let attributes = attributes(&exporter, &provider, "open_managed_session_inner");+ assert_eq!(+ attributes.get("agent.session.warm_miss_reason"),+ Some(&opentelemetry::Value::from("no_warm_id"))+ );+ assert_eq!(+ attributes.get("agent.session.warm_id_sent"),+ Some(&opentelemetry::Value::Bool(false))+ );+}++#[tokio::test]+async fn warming_past_the_owner_cap_records_the_refusal() {+ let (exporter, provider, _guard) = span_exporter();+ let (service, _, _, _, _) = harness();+ let service = service.with_warm_sessions(+ Arc::new(WarmLifecycleStub {+ claim: WarmClaim::Claimed,+ }),+ Arc::new(agent_session::domain::ports::NoOpToolCatalog),+ );+ let owner = model_owner::Owner::User(sender());+ {+ let mut reservations = service.warm_reservations.lock().await;+ for _ in 0..2 {+ reservations.insert(+ AgentSessionId::new(),+ (owner.clone(), std::time::Instant::now()),+ );+ }+ }+ assert!(+ service+ .warm_session(owner, AgentSessionId::new())+ .await+ .unwrap()+ .is_none()+ );++ let attributes = attributes(&exporter, &provider, "agent.session.prepare_warm");+ assert_eq!(+ attributes.get("agent.warm.outcome"),+ Some(&opentelemetry::Value::from("refused_owner_cap"))+ );+ assert!(attributes.contains_key("agent.session.model"));+}diff --git a/crates/agent_harness/src/domain/service/warm.rs b/crates/agent_harness/src/domain/service/warm.rsindex 0c521020f9..8b2c4cb1e4 100644--- a/crates/agent_harness/src/domain/service/warm.rs+++ b/crates/agent_harness/src/domain/service/warm.rs@@ -4,6 +4,7 @@ use super::*;use crate::domain::model::SessionRepository;use agent_session::domain::model::session_owner_user;use agent_session::domain::ports::{OpenManagedSession, SelectedManagedPersona};+use agent_session::domain::warm::WarmClaim;use model_owner::Owner;use std::time::{Duration, Instant};@@ -51,7 +52,7 @@ whereMentions: PromptMentions,Notifier: AgentSessionNotifier,{- #[tracing::instrument(skip_all, err, fields(agent.session.id = tracing::field::Empty, agent.harness = tracing::field::Empty, agent.session.warm = warm, agent.session.warm_hit = false))]+ #[tracing::instrument(skip_all, err, fields(agent.session.id = tracing::field::Empty, agent.harness = tracing::field::Empty, agent.session.warm = warm, agent.session.warm_hit = false, agent.session.warm_id_sent = tracing::field::Empty, agent.session.warm_miss_reason = tracing::field::Empty, agent.session.model = tracing::field::Empty))]pub(super) async fn open_managed_session_inner(&self,request: agent_session::domain::ports::OpenManagedSession,@@ -96,6 +97,7 @@ where// in is choosing what this session runs on, for its whole life.let requested_model = request.model;let model = requested_model.clone().unwrap_or(model);+ tracing::Span::current().record("agent.session.model", &model);let kind = AgentKind::for_session(bot_id, &harness);let harness = kind.harness_slug().map_or(harness, str::to_owned);tracing::Span::current().record("agent.harness", &harness);@@ -154,22 +156,31 @@ where.instrument(tracing::info_span!("agent.init.admission", agent.session.id = %session_id)).await.map_err(into_session_error)?;- if !warm- && kind == AgentKind::InMemory- && bot_id == bot_id::MACRO_NEW_BOT_ID- && matches!(mcp_servers, AgentMcpServers::OwnerConnections)- && request.repo_url.is_none()- && let Some(lifecycle) = &self.warm_lifecycle- && lifecycle- .claim(- session_id,- &request.owner,- bot_id,- &model,- instructions.as_deref(),- )- .await?- {+ let claim = if warm {+ None+ } else {+ let eligible = kind == AgentKind::InMemory+ && bot_id == bot_id::MACRO_NEW_BOT_ID+ && matches!(mcp_servers, AgentMcpServers::OwnerConnections)+ && request.repo_url.is_none();+ let claim = match &self.warm_lifecycle {+ Some(lifecycle) if eligible => Some(+ lifecycle+ .claim(+ session_id,+ &request.owner,+ bot_id,+ &model,+ instructions.as_deref(),+ )+ .await?,+ ),+ _ => None,+ };+ record_warm_claim(request.id.is_some(), eligible, claim);+ claim+ };+ if claim == Some(WarmClaim::Claimed) {self.warm_reservations.lock().await.remove(&session_id);tracing::Span::current().record("agent.session.warm_hit", true);let session = self.inner.sessions.get_session(session_id).await?;@@ -357,28 +368,40 @@ whereself}+ #[tracing::instrument(name = "agent.session.prepare_warm", skip_all, fields(agent.session.id = %id, agent.warm.outcome = tracing::field::Empty, agent.session.model = tracing::field::Empty))]pub(super) async fn prepare_warm(&self,owner: Owner,id: AgentSessionId,) -> agent_session::domain::error::Result<Option<AgentSession>> {+ let span = tracing::Span::current();session_owner_user(&owner)?;+ span.record(+ "agent.session.model",+ &self.inner.defaults.for_bot(bot_id::MACRO_NEW_BOT_ID).model,+ );if self.warm_lifecycle.is_none() {+ span.record("agent.warm.outcome", "disabled");return Ok(None);}{let mut reservations = self.warm_reservations.lock().await;reservations.retain(|_, (_, started)| started.elapsed() < WARM_RESERVATION_TTL);if reservations.contains_key(&id) {+ span.record("agent.warm.outcome", "reused");+ return Ok(None);+ }+ if reservations.len() >= MAX_WARM_RESERVATIONS {+ span.record("agent.warm.outcome", "refused_global_cap");return Ok(None);}- if reservations.len() >= MAX_WARM_RESERVATIONS- || reservations- .values()- .filter(|(user, _)| user == &owner)- .count()- >= MAX_WARM_RESERVATIONS_PER_OWNER+ if reservations+ .values()+ .filter(|(user, _)| user == &owner)+ .count()+ >= MAX_WARM_RESERVATIONS_PER_OWNER{+ span.record("agent.warm.outcome", "refused_owner_cap");return Ok(None);}reservations.insert(id, (owner.clone(), Instant::now()));@@ -402,12 +425,17 @@ where).await{- Ok(Ok(session)) => Ok(Some(session)),+ Ok(Ok(session)) => {+ span.record("agent.warm.outcome", "prepared");+ Ok(Some(session))+ }Ok(Err(error)) => {+ span.record("agent.warm.outcome", "failed");tracing::warn!(error = ?error, "warm session preparation failed");Ok(None)}Err(_) => {+ span.record("agent.warm.outcome", "timed_out");tracing::warn!("warm session preparation timed out");Ok(None)}@@ -432,3 +460,21 @@ where}}}++/// Why a non-warming open did or did not claim a warm session, on its span.+fn record_warm_claim(client_minted_id: bool, eligible: bool, claim: Option<WarmClaim>) {+ let span = tracing::Span::current();+ let reason: &'static str = match claim {+ _ if !client_minted_id => "no_warm_id",+ Some(WarmClaim::Claimed) => "none",+ Some(WarmClaim::Missed(reason)) => reason.into(),+ None if !eligible => "ineligible",+ None => "disabled",+ };+ span.record("agent.session.warm_miss_reason", reason);+ if !client_minted_id {+ span.record("agent.session.warm_id_sent", false);+ } else if let Some(claim) = claim {+ span.record("agent.session.warm_id_sent", claim.found_warm_session());+ }+}diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rsindex 9c1ea771cb..242fcca82f 100644--- a/crates/agent_session/src/domain/service.rs+++ b/crates/agent_session/src/domain/service.rs@@ -592,6 +592,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {agent.action.name = action.as_ref(),agent.command.queue_wait_ms = tracing::field::Empty,agent.session.runtime_phase_at_dequeue = tracing::field::Empty,+ agent.command.handshake_wait_ms = tracing::field::Empty,otel.status_code = tracing::field::Empty,otel.status_description = tracing::field::Empty,);diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rsindex e40c6bb733..c5443593f1 100644--- a/crates/agent_session/src/domain/service/test.rs+++ b/crates/agent_session/src/domain/service/test.rs@@ -2161,6 +2161,157 @@ async fn a_prompt_turn_is_traced_as_an_agent_span_under_its_command() {assert!(input.contains("what time is it?"), "{input}");}+/// A command dequeued before the handshake finished says how long it waited+/// for it; one dequeued on a live session waited for nothing.+#[tokio::test]+async fn a_command_span_records_its_wait_for_the_handshake() {+ use opentelemetry::trace::TracerProvider as _;+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};+ use tracing_subscriber::layer::SubscriberExt as _;++ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));+ let _guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ tracing::callsite::rebuild_interest_cache();++ let repo = InMemoryAgentSessionRepo::new();+ let session = test_session();+ repo.insert_session(test_agent_session(session));+ let entered = Arc::new(Notify::new());+ let release = Arc::new(Notify::new());+ let logs = BlockingPromptLogs {+ repo: repo.clone(),+ entered: entered.clone(),+ release: release.clone(),+ hang_disconnect: false,+ fail_restore_log: None,+ };+ let (outbound_tx, mut outbound_rx) = mpsc::channel(8);+ let (inbound_tx, inbound_rx) = mpsc::channel(8);+ let (commands, command_rx) = mpsc::channel(8);+ let (handshake, _) = watch::channel(HandshakeStatus::Pending);+ let actor = SessionActor::new(+ session,+ None,+ "/workspace".to_owned(),+ Vec::new(),+ crate::domain::session::PermissionPolicy::AutoAccept,+ RecordingTransport {+ outbound: outbound_tx,+ inbound: inbound_rx,+ },+ logs,+ command_rx,+ handshake,+ Arc::new(crate::domain::ports::NoOpTurnObserver),+ Arc::new(crate::domain::ports::NoOpToolCatalog),+ );+ let active = Arc::new(ActiveSessions::new());+ let (stopped_tx, _) = watch::channel(false);+ let claim = claim_for_test(&repo, session).await;+ let task = tokio::spawn(+ run_session(+ actor,+ Arc::downgrade(&active),+ Arc::new(()),+ stopped_tx,+ CancellationToken::new(),+ repo.clone(),+ claim,+ Arc::new(crate::domain::ports::NoOpTurnObserver),+ )+ .with_current_subscriber(),+ );+ let send_prompt = |name: &'static str| {+ let (completed, result) = oneshot::channel();+ let command = SessionCommand {+ user_id: None,+ action: AgentAction::prompt(name),+ action_id: AgentActionId::mint(),+ completed,+ span: tracing::info_span!(+ "agent.session.command",+ agent.command.name = name,+ agent.command.queue_wait_ms = tracing::field::Empty,+ agent.session.runtime_phase_at_dequeue = tracing::field::Empty,+ agent.command.handshake_wait_ms = tracing::field::Empty,+ ),+ enqueued_at: tokio::time::Instant::now(),+ };+ (command, result)+ };++ // Queued before the runtime is even ready: it waits out the handshake.+ let (early, early_result) = send_prompt("early");+ commands.send(early).await.unwrap();+ open_test_session(&inbound_tx, &mut outbound_rx, session).await;+ entered.notified().await;+ release.notify_one();+ outbound_rx+ .recv()+ .await+ .expect("early prompt is dispatched");+ early_result+ .await+ .unwrap()+ .expect("early delivery completes");++ let (live, live_result) = send_prompt("live");+ commands.send(live).await.unwrap();+ entered.notified().await;+ release.notify_one();+ outbound_rx.recv().await.expect("live prompt is dispatched");+ live_result.await.unwrap().expect("live delivery completes");++ drop(inbound_tx);+ task.await.unwrap();+ provider.force_flush().expect("flush");+ let spans = exporter.get_finished_spans().expect("finished spans");+ let attribute = |name: &str, key: &str| {+ let span = spans+ .iter()+ .find(|span| {+ span.name == "agent.session.command"+ && span.attributes.iter().any(|kv| {+ kv.key.as_str() == "agent.command.name" && kv.value.as_str() == name+ })+ })+ .unwrap_or_else(|| panic!("no {name} command span: {spans:#?}"));+ span.attributes+ .iter()+ .find(|kv| kv.key.as_str() == key)+ .map(|kv| kv.value.clone())+ };+ assert!(matches!(+ attribute("early", "agent.session.runtime_phase_at_dequeue")+ .expect("phase recorded")+ .as_str()+ .as_ref(),+ "booting" | "handshaking"+ ));+ assert!(matches!(+ attribute("early", "agent.command.handshake_wait_ms"),+ Some(opentelemetry::Value::I64(wait)) if wait >= 0+ ));+ assert_eq!(+ attribute("live", "agent.session.runtime_phase_at_dequeue")+ .expect("phase recorded")+ .as_str(),+ "live"+ );+ assert_eq!(+ attribute("live", "agent.command.handshake_wait_ms"),+ Some(opentelemetry::Value::I64(0))+ );+ assert!(matches!(+ attribute("live", "agent.command.queue_wait_ms"),+ Some(opentelemetry::Value::I64(_))+ ));+}+mod initial_model;mod owner_binding;mod turn_projection;diff --git a/crates/agent_session/src/domain/session/actors.rs b/crates/agent_session/src/domain/session/actors.rsindex a37dbfb938..1924fb3935 100644--- a/crates/agent_session/src/domain/session/actors.rs+++ b/crates/agent_session/src/domain/session/actors.rs@@ -62,6 +62,8 @@ pub(crate) struct SessionCommand {pub(crate) struct SessionCompletion {completed: oneshot::Sender<Result<()>>,span: tracing::Span,+ /// When the command was dequeued, if the runtime was not live yet.+ dequeued_before_live: Option<Instant>,}/// Whether a [`SessionActor`] has more steps to take.@@ -238,17 +240,17 @@ whereSome(SessionCommand { user_id, action, action_id, completed, span, enqueued_at }) => {span.record("agent.command.queue_wait_ms",- enqueued_at.elapsed().as_millis() as u64,- );- span.record(- "agent.session.runtime_phase_at_dequeue",- self.machine.status().as_ref(),+ elapsed_ms(enqueued_at),);+ let status = self.machine.status();+ span.record("agent.session.runtime_phase_at_dequeue", status.as_ref());+ let dequeued_before_live = (!matches!(status, RuntimeStatus::Live { .. }))+ .then(Instant::now);Input::Command {from: user_id,action,action_id,- token: SessionCompletion { completed, span },+ token: SessionCompletion { completed, span, dequeued_before_live },}},// The service dropped every handle; nobody can reach us.@@ -301,7 +303,13 @@ wherematch effect {Effect::Send { from, mut message } => {let command_span = effects.front().and_then(|effect| match effect {- Effect::Complete { token, .. } => Some(token.span.clone()),+ Effect::Complete { token, .. } => {+ token.span.record(+ "agent.command.handshake_wait_ms",+ token.dequeued_before_live.map_or(0, elapsed_ms),+ );+ Some(token.span.clone())+ }_ => None,});// Before delivery, so the turn's span starts when the@@ -732,6 +740,11 @@ where}}+/// Milliseconds as i64: a u64 reaches OTel as a string.+fn elapsed_ms(since: Instant) -> i64 {+ i64::try_from(since.elapsed().as_millis()).unwrap_or(i64::MAX)+}+fn acp_method(message: &ToRuntimeMessage) -> Option<&str> {let ToRuntimeMessage::Acp(AcpMessage(frame)) = message else {return None;diff --git a/crates/agent_session/src/domain/warm.rs b/crates/agent_session/src/domain/warm.rsindex ae5901e89a..5bbc00af62 100644--- a/crates/agent_session/src/domain/warm.rs+++ b/crates/agent_session/src/domain/warm.rs@@ -6,6 +6,41 @@ use model_owner::Owner;use std::future::Future;use std::pin::Pin;+/// What a claim found; a miss says the first claim condition that failed.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum WarmClaim {+ Claimed,+ Missed(WarmMissReason),+}++/// Why a claim missed, in the order the claim checks them.+#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)]+#[strum(serialize_all = "snake_case")]+pub enum WarmMissReason {+ /// No session has this id: the client sent a fresh id, or the warm row is gone.+ NotFound,+ /// The session is visible or belongs to a thread.+ NotWarm,+ OwnerMismatch,+ BotMismatch,+ ModelMismatch,+ InstructionsMismatch,+ Disconnected,+ Expired,+ /// Every condition held, but a concurrent claim promoted it first.+ LostRace,+}++impl WarmClaim {+ /// Whether the id named a warm session at all.+ pub fn found_warm_session(self) -> bool {+ !matches!(+ self,+ Self::Missed(WarmMissReason::NotFound | WarmMissReason::NotWarm)+ )+ }+}+/// Persistence operations whose claim and expiry must serialize across replicas.pub trait WarmSessionLifecycle: Send + Sync {/// Reveal only a fresh warm session belonging to this owner and exact persona.@@ -16,7 +51,7 @@ pub trait WarmSessionLifecycle: Send + Sync {bot: BotId,model: &'a str,instructions: Option<&'a str>,- ) -> Pin<Box<dyn Future<Output = Result<bool>> + Send + 'a>>;+ ) -> Pin<Box<dyn Future<Output = Result<WarmClaim>> + Send + 'a>>;/// Mark expired hidden sessions disconnected, returning ids for routed deletion./// Claimed sessions are never selected. Work is bounded per sweep.fn expire(&self) -> Pin<Box<dyn Future<Output = Result<Vec<AgentSessionId>>> + Send + '_>>;diff --git a/crates/agent_session/src/outbound/postgres/test.rs b/crates/agent_session/src/outbound/postgres/test.rsindex de15e7adbb..6c38be1445 100644--- a/crates/agent_session/src/outbound/postgres/test.rs+++ b/crates/agent_session/src/outbound/postgres/test.rs@@ -3009,7 +3009,7 @@ async fn the_originating_channel_steers_only_shared_agents_sessions(pool: PgPool#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPool) {- use crate::domain::warm::WarmSessionLifecycle;+ use crate::domain::warm::{WarmClaim, WarmMissReason, WarmSessionLifecycle};let bot = create_test_bot(&pool).await;let repo = test_repo(&pool);let mut params = new_session(bot, None, None);@@ -3024,13 +3024,14 @@ async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPoo.is_empty());let other = Owner::User(user_id("macro|[email protected]"));- assert!(- !WarmSessionLifecycle::claim(&repo, session.id, &other, bot, &session.model, None)+ assert_eq!(+ WarmSessionLifecycle::claim(&repo, session.id, &other, bot, &session.model, None).await- .unwrap()+ .unwrap(),+ WarmClaim::Missed(WarmMissReason::OwnerMismatch));- assert!(- !WarmSessionLifecycle::claim(+ assert_eq!(+ WarmSessionLifecycle::claim(&repo,session.id,&session.owner_id,@@ -3039,10 +3040,11 @@ async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPooNone).await- .unwrap()+ .unwrap(),+ WarmClaim::Missed(WarmMissReason::ModelMismatch));- assert!(- !WarmSessionLifecycle::claim(+ assert_eq!(+ WarmSessionLifecycle::claim(&repo,session.id,&session.owner_id,@@ -3051,7 +3053,21 @@ async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPooSome("different instructions")).await- .unwrap()+ .unwrap(),+ WarmClaim::Missed(WarmMissReason::InstructionsMismatch)+ );+ assert_eq!(+ WarmSessionLifecycle::claim(+ &repo,+ AgentSessionId::new(),+ &session.owner_id,+ bot,+ &session.model,+ None+ )+ .await+ .unwrap(),+ WarmClaim::Missed(WarmMissReason::NotFound));let (left, right) = tokio::join!(WarmSessionLifecycle::claim(@@ -3071,7 +3087,14 @@ async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPooNone),);- assert_ne!(left.unwrap(), right.unwrap());+ let mut outcomes = [left.unwrap(), right.unwrap()];+ outcomes.sort_by_key(|claim| *claim != WarmClaim::Claimed);+ // The loser either waited on the winner's row lock or read the claimed row.+ assert_eq!(outcomes[0], WarmClaim::Claimed);+ assert!(matches!(+ outcomes[1],+ WarmClaim::Missed(WarmMissReason::LostRace | WarmMissReason::NotWarm)+ ));assert_eq!(repo.recent_for_owner(&user_id(OWNER), NonZeroUsize::new(10).unwrap()).await@@ -3084,7 +3107,7 @@ async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPoo#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn expired_warm_sessions_cannot_be_claimed(pool: PgPool) {- use crate::domain::warm::WarmSessionLifecycle;+ use crate::domain::warm::{WarmClaim, WarmMissReason, WarmSessionLifecycle};let bot = create_test_bot(&pool).await;let repo = test_repo(&pool);let mut params = new_session(bot, None, None);@@ -3097,9 +3120,22 @@ async fn expired_warm_sessions_cannot_be_claimed(pool: PgPool) {.execute(&pool).await.unwrap();+ assert_eq!(+ WarmSessionLifecycle::claim(+ &repo,+ session.id,+ &session.owner_id,+ bot,+ &session.model,+ None+ )+ .await+ .unwrap(),+ WarmClaim::Missed(WarmMissReason::Expired)+ );assert_eq!(repo.expire().await.unwrap(), vec![session.id]);- assert!(- !WarmSessionLifecycle::claim(+ assert_eq!(+ WarmSessionLifecycle::claim(&repo,session.id,&session.owner_id,@@ -3108,7 +3144,8 @@ async fn expired_warm_sessions_cannot_be_claimed(pool: PgPool) {None).await- .unwrap()+ .unwrap(),+ WarmClaim::Missed(WarmMissReason::Disconnected));AgentSessionRepo::delete(&repo, session.id).await.unwrap();assert!(repo.expire().await.unwrap().is_empty());diff --git a/crates/agent_session/src/outbound/postgres/warm.rs b/crates/agent_session/src/outbound/postgres/warm.rsindex 3e891fb8f9..82613d31df 100644--- a/crates/agent_session/src/outbound/postgres/warm.rs+++ b/crates/agent_session/src/outbound/postgres/warm.rs@@ -1,6 +1,6 @@//! Warm sessions use list_hidden without a thread; hidden mentions always have a thread.use super::*;-use crate::domain::warm::WarmSessionLifecycle;+use crate::domain::warm::{WarmClaim, WarmMissReason, WarmSessionLifecycle};use std::{future::Future, pin::Pin};impl<B: BotFacts + 'static> WarmSessionLifecycle for PgAgentSessionRepo<B> {@@ -11,19 +11,46 @@ impl<B: BotFacts + 'static> WarmSessionLifecycle for PgAgentSessionRepo<B> {bot: BotId,model: &'a str,instructions: Option<&'a str>,- ) -> Pin<Box<dyn Future<Output = Result<bool>> + Send + 'a>> {+ ) -> Pin<Box<dyn Future<Output = Result<WarmClaim>> + Send + 'a>> {Box::pin(async move {let user = session_owner_user(owner)?;let mut transaction = self.pool.begin().await.context("begin warm claim")?;- let changed = sqlx::query!(+ // One statement: the snapshot that explains a miss is the one the claim ran against.+ let row = sqlx::query!(r#"- UPDATE agent_session SET list_hidden = false- WHERE id = $1 AND owner_id = $2 AND bot_id = $3 AND model = $4- AND instructions IS NOT DISTINCT FROM $5- AND list_hidden AND thread_id IS NULL- AND status != 'disconnected'- AND NOT (status = 'event' AND status_event_name = 'disconnected')- AND created_at > NOW() - INTERVAL '10 minutes'+ WITH candidate AS (+ SELECT+ true AS found,+ list_hidden AND thread_id IS NULL AS warm,+ owner_id = $2 AS owner_matches,+ bot_id = $3 AS bot_matches,+ model = $4 AS model_matches,+ instructions IS NOT DISTINCT FROM $5 AS instructions_match,+ COALESCE(status != 'disconnected'+ AND NOT (status = 'event' AND status_event_name = 'disconnected'), false) AS live,+ created_at > NOW() - INTERVAL '10 minutes' AS fresh+ FROM agent_session WHERE id = $1+ ), claimed AS (+ UPDATE agent_session SET list_hidden = false+ WHERE id = $1 AND owner_id = $2 AND bot_id = $3 AND model = $4+ AND instructions IS NOT DISTINCT FROM $5+ AND list_hidden AND thread_id IS NULL+ AND status != 'disconnected'+ AND NOT (status = 'event' AND status_event_name = 'disconnected')+ AND created_at > NOW() - INTERVAL '10 minutes'+ RETURNING id+ )+ SELECT+ EXISTS (SELECT 1 FROM claimed) AS "claimed!",+ candidate.found AS "found?",+ candidate.warm AS "warm?",+ candidate.owner_matches AS "owner_matches?",+ candidate.bot_matches AS "bot_matches?",+ candidate.model_matches AS "model_matches?",+ candidate.instructions_match AS "instructions_match?",+ candidate.live AS "live?",+ candidate.fresh AS "fresh?"+ FROM (SELECT 1) AS one LEFT JOIN candidate ON true"#,id.as_uuid(),user.as_ref(),@@ -31,18 +58,37 @@ impl<B: BotFacts + 'static> WarmSessionLifecycle for PgAgentSessionRepo<B> {model,instructions)- .execute(&mut *transaction)+ .fetch_one(&mut *transaction).await- .context("claim warm session")?- .rows_affected()- == 1;- if changed {+ .context("claim warm session")?;+ let checks = [+ (row.warm, WarmMissReason::NotWarm),+ (row.owner_matches, WarmMissReason::OwnerMismatch),+ (row.bot_matches, WarmMissReason::BotMismatch),+ (row.model_matches, WarmMissReason::ModelMismatch),+ (row.instructions_match, WarmMissReason::InstructionsMismatch),+ (row.live, WarmMissReason::Disconnected),+ (row.fresh, WarmMissReason::Expired),+ ];+ let claim = if row.claimed {+ WarmClaim::Claimed+ } else if row.found.is_none() {+ WarmClaim::Missed(WarmMissReason::NotFound)+ } else {+ WarmClaim::Missed(+ checks+ .into_iter()+ .find_map(|(held, reason)| (held != Some(true)).then_some(reason))+ .unwrap_or(WarmMissReason::LostRace),+ )+ };+ if claim == WarmClaim::Claimed {upsert_user_history(&mut transaction, user.as_ref(), &id.as_uuid()).await.context("record claimed warm session history")?;}transaction.commit().await.context("commit warm claim")?;- Ok(changed)+ Ok(claim)})}fn expire(&self) -> Pin<Box<dyn Future<Output = Result<Vec<AgentSessionId>>> + Send + '_>> {diff --git a/deny.toml b/deny.tomlindex ccf1225603..5217537605 100644--- a/deny.toml+++ b/deny.toml@@ -201,6 +201,7 @@ deny = [], reason = "Use macro_aws_config for AWS configuration in workspace crates" },{ crate = "tracing-subscriber", wrappers = ["agent", # GenAI span tests+ "agent_harness", # Warm-claim and handshake span tests"agent_inmem", # GenAI trace propagation tests"agent_session", # GenAI span tests"ai_toolset", # GenAI span tests