Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(observability): warm-claim miss reasons and handshake wait (#7748)

Wolf Mermelsteind3e3eda9e094
diff --git a/.sqlx/query-1e2b5f16fc1eca3b9f8261470249db35499a25f4251d8b62d7ae5a91ad7b7182.json b/.sqlx/query-1e2b5f16fc1eca3b9f8261470249db35499a25f4251d8b62d7ae5a91ad7b7182.json
new file mode 100644
index 0000000000..a898f659bb
--- /dev/null
+++ b/.sqlx/query-1e2b5f16fc1eca3b9f8261470249db35499a25f4251d8b62d7ae5a91ad7b7182.json
@@ -0,0 +1,74 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH candidate AS (\n SELECT\n true AS found,\n list_hidden AND thread_id IS NULL AS warm,\n owner_id = $2 AS owner_matches,\n bot_id = $3 AS bot_matches,\n model = $4 AS model_matches,\n instructions IS NOT DISTINCT FROM $5 AS instructions_match,\n COALESCE(status != 'disconnected'\n AND NOT (status = 'event' AND status_event_name = 'disconnected'), false) AS live,\n created_at > NOW() - INTERVAL '10 minutes' AS fresh\n FROM agent_session WHERE id = $1\n ), claimed AS (\n UPDATE agent_session SET list_hidden = false\n WHERE id = $1 AND owner_id = $2 AND bot_id = $3 AND model = $4\n AND instructions IS NOT DISTINCT FROM $5\n AND list_hidden AND thread_id IS NULL\n AND status != 'disconnected'\n AND NOT (status = 'event' AND status_event_name = 'disconnected')\n AND created_at > NOW() - INTERVAL '10 minutes'\n RETURNING id\n )\n SELECT\n EXISTS (SELECT 1 FROM claimed) AS \"claimed!\",\n candidate.found AS \"found?\",\n candidate.warm AS \"warm?\",\n candidate.owner_matches AS \"owner_matches?\",\n candidate.bot_matches AS \"bot_matches?\",\n candidate.model_matches AS \"model_matches?\",\n candidate.instructions_match AS \"instructions_match?\",\n candidate.live AS \"live?\",\n candidate.fresh AS \"fresh?\"\n FROM (SELECT 1) AS one LEFT JOIN candidate ON true\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "claimed!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 1,
+ "name": "found?",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 2,
+ "name": "warm?",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 3,
+ "name": "owner_matches?",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 4,
+ "name": "bot_matches?",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 5,
+ "name": "model_matches?",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 6,
+ "name": "instructions_match?",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 7,
+ "name": "live?",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 8,
+ "name": "fresh?",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Text",
+ "Uuid",
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ ]
+ },
+ "hash": "1e2b5f16fc1eca3b9f8261470249db35499a25f4251d8b62d7ae5a91ad7b7182"
+}
diff --git a/.sqlx/query-7b29542761b4999a6e3525de2a86047f8612c24d28aeb03e777c024dd9a95938.json b/.sqlx/query-7b29542761b4999a6e3525de2a86047f8612c24d28aeb03e777c024dd9a95938.json
deleted file mode 100644
index eb76391d9a..0000000000
--- a/.sqlx/query-7b29542761b4999a6e3525de2a86047f8612c24d28aeb03e777c024dd9a95938.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n UPDATE agent_session SET list_hidden = false\n WHERE id = $1 AND owner_id = $2 AND bot_id = $3 AND model = $4\n AND instructions IS NOT DISTINCT FROM $5\n AND list_hidden AND thread_id IS NULL\n AND status != 'disconnected'\n AND NOT (status = 'event' AND status_event_name = 'disconnected')\n AND created_at > NOW() - INTERVAL '10 minutes'\n ",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid",
- "Text",
- "Uuid",
- "Text",
- "Text"
- ]
- },
- "nullable": []
- },
- "hash": "7b29542761b4999a6e3525de2a86047f8612c24d28aeb03e777c024dd9a95938"
-}
diff --git a/Cargo.lock b/Cargo.lock
index fab57d3179..c3f5e41c94 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -373,6 +373,7 @@ dependencies = [
"model_user",
"notification",
"opentelemetry",
+ "opentelemetry_sdk",
"pipedream_mcp",
"redis",
"reqwest 0.13.4",
@@ -389,6 +390,7 @@ dependencies = [
"tower 0.5.3",
"tracing",
"tracing-opentelemetry",
+ "tracing-subscriber",
"url",
"utoipa",
"uuid",
diff --git a/crates/agent_harness/Cargo.toml b/crates/agent_harness/Cargo.toml
index ebee8170cf..8cd1213c4e 100644
--- a/crates/agent_harness/Cargo.toml
+++ b/crates/agent_harness/Cargo.toml
@@ -91,5 +91,7 @@ bot_id = { path = "../bot_id", features = ["test-utils"] }
harness_id = { path = "../harness_id", features = ["test-utils"] }
mockall = { workspace = true }
model_user = { path = "../model_user" }
+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }
tower = { workspace = true }
+tracing-subscriber = { workspace = true }
wiremock = { workspace = true }
diff --git a/crates/agent_harness/src/domain/service/test/warm.rs b/crates/agent_harness/src/domain/service/test/warm.rs
index 0c028541f2..1f70fd287c 100644
--- a/crates/agent_harness/src/domain/service/test/warm.rs
+++ b/crates/agent_harness/src/domain/service/test/warm.rs
@@ -1,7 +1,8 @@
use super::*;
+use agent_session::domain::warm::{WarmClaim, WarmMissReason};
struct WarmLifecycleStub {
- claim: bool,
+ claim: WarmClaim,
}
impl agent_session::domain::warm::WarmSessionLifecycle for WarmLifecycleStub {
fn claim<'a>(
@@ -13,7 +14,7 @@ impl agent_session::domain::warm::WarmSessionLifecycle for WarmLifecycleStub {
_instructions: Option<&'a str>,
) -> std::pin::Pin<
Box<
- dyn std::future::Future<Output = agent_session::domain::error::Result<bool>>
+ dyn std::future::Future<Output = agent_session::domain::error::Result<WarmClaim>>
+ Send
+ 'a,
>,
@@ -38,7 +39,9 @@ impl agent_session::domain::warm::WarmSessionLifecycle for WarmLifecycleStub {
async fn warming_rejects_bot_owners_and_bounds_speculation_per_user() {
let (service, _, containers, _, _) = harness();
let service = service.with_warm_sessions(
- Arc::new(WarmLifecycleStub { claim: false }),
+ Arc::new(WarmLifecycleStub {
+ claim: WarmClaim::Missed(WarmMissReason::NotFound),
+ }),
Arc::new(agent_session::domain::ports::NoOpToolCatalog),
);
assert!(matches!(
@@ -95,8 +98,12 @@ async fn warming_initializes_without_a_prompt_or_visible_created_event() {
let ((service, repo, containers, _, _), signals) =
harness_with_signals(PromptContextMock::default(), PromptComposerMock::default());
let tools = Arc::new(GatedWarmTools(tokio::sync::Semaphore::new(0)));
- let service =
- service.with_warm_sessions(Arc::new(WarmLifecycleStub { claim: true }), tools.clone());
+ let service = service.with_warm_sessions(
+ Arc::new(WarmLifecycleStub {
+ claim: WarmClaim::Claimed,
+ }),
+ tools.clone(),
+ );
let id = AgentSessionId::new();
let mut warm = Box::pin(service.warm_session(model_owner::Owner::User(sender()), id));
let drive = async {
@@ -152,3 +159,172 @@ async fn warming_initializes_without_a_prompt_or_visible_created_event() {
"claim reveals the conversation"
);
}
+
+fn span_exporter() -> (
+ opentelemetry_sdk::trace::InMemorySpanExporter,
+ opentelemetry_sdk::trace::SdkTracerProvider,
+ tracing::subscriber::DefaultGuard,
+) {
+ use opentelemetry::trace::TracerProvider as _;
+ use tracing_subscriber::layer::SubscriberExt as _;
+ let exporter = opentelemetry_sdk::trace::InMemorySpanExporter::default();
+ let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));
+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ tracing::callsite::rebuild_interest_cache();
+ (exporter, provider, guard)
+}
+
+fn attributes(
+ exporter: &opentelemetry_sdk::trace::InMemorySpanExporter,
+ provider: &opentelemetry_sdk::trace::SdkTracerProvider,
+ name: &str,
+) -> std::collections::HashMap<String, opentelemetry::Value> {
+ provider.force_flush().expect("flush");
+ let spans = exporter.get_finished_spans().expect("finished spans");
+ let span = spans
+ .iter()
+ .find(|span| span.name == name)
+ .unwrap_or_else(|| {
+ panic!(
+ "no {name} span in {:?}",
+ spans.iter().map(|span| &span.name).collect::<Vec<_>>()
+ )
+ });
+ span.attributes
+ .iter()
+ .map(|kv| (kv.key.to_string(), kv.value.clone()))
+ .collect()
+}
+
+#[tokio::test]
+async fn a_missed_warm_claim_records_why_on_the_open_span() {
+ let (exporter, provider, _guard) = span_exporter();
+ let (service, _, _, _, _) = harness();
+ let service = service.with_warm_sessions(
+ Arc::new(WarmLifecycleStub {
+ claim: WarmClaim::Missed(WarmMissReason::ModelMismatch),
+ }),
+ Arc::new(agent_session::domain::ports::NoOpToolCatalog),
+ );
+ let session = service
+ .open_managed_session(OpenManagedSession {
+ id: Some(AgentSessionId::new()),
+ owner: model_owner::Owner::User(sender()),
+ prompt: None,
+ repo_url: None,
+ repo_branch: None,
+ instructions: None,
+ model: Some("composer-model".to_owned()),
+ profile: Some(agent_session::domain::ports::SelectedManagedPersona {
+ bot_id: bot_id::MACRO_NEW_BOT_ID,
+ profile: None,
+ }),
+ })
+ .await
+ .expect("cold open after a missed claim");
+ // The session's actor holds the open span until the session ends.
+ service
+ .execute(session.id, HarnessCommand::Delete)
+ .await
+ .expect("delete");
+
+ let attributes = attributes(&exporter, &provider, "open_managed_session_inner");
+ assert_eq!(
+ attributes.get("agent.session.warm_miss_reason"),
+ Some(&opentelemetry::Value::from("model_mismatch"))
+ );
+ assert_eq!(
+ attributes.get("agent.session.warm_id_sent"),
+ Some(&opentelemetry::Value::Bool(true))
+ );
+ assert_eq!(
+ attributes.get("agent.session.warm_hit"),
+ Some(&opentelemetry::Value::Bool(false))
+ );
+ assert_eq!(
+ attributes.get("agent.session.model"),
+ Some(&opentelemetry::Value::from("composer-model"))
+ );
+}
+
+#[tokio::test]
+async fn an_open_without_a_client_id_records_that_no_warm_id_was_sent() {
+ let (exporter, provider, _guard) = span_exporter();
+ let (service, _, _, _, _) = harness();
+ let service = service.with_warm_sessions(
+ Arc::new(WarmLifecycleStub {
+ claim: WarmClaim::Missed(WarmMissReason::NotFound),
+ }),
+ Arc::new(agent_session::domain::ports::NoOpToolCatalog),
+ );
+ let session = service
+ .open_managed_session(OpenManagedSession {
+ id: None,
+ owner: model_owner::Owner::User(sender()),
+ prompt: None,
+ repo_url: None,
+ repo_branch: None,
+ instructions: None,
+ model: None,
+ profile: Some(agent_session::domain::ports::SelectedManagedPersona {
+ bot_id: bot_id::MACRO_NEW_BOT_ID,
+ profile: None,
+ }),
+ })
+ .await
+ .expect("cold open");
+ // The session's actor holds the open span until the session ends.
+ service
+ .execute(session.id, HarnessCommand::Delete)
+ .await
+ .expect("delete");
+
+ let attributes = attributes(&exporter, &provider, "open_managed_session_inner");
+ assert_eq!(
+ attributes.get("agent.session.warm_miss_reason"),
+ Some(&opentelemetry::Value::from("no_warm_id"))
+ );
+ assert_eq!(
+ attributes.get("agent.session.warm_id_sent"),
+ Some(&opentelemetry::Value::Bool(false))
+ );
+}
+
+#[tokio::test]
+async fn warming_past_the_owner_cap_records_the_refusal() {
+ let (exporter, provider, _guard) = span_exporter();
+ let (service, _, _, _, _) = harness();
+ let service = service.with_warm_sessions(
+ Arc::new(WarmLifecycleStub {
+ claim: WarmClaim::Claimed,
+ }),
+ Arc::new(agent_session::domain::ports::NoOpToolCatalog),
+ );
+ let owner = model_owner::Owner::User(sender());
+ {
+ let mut reservations = service.warm_reservations.lock().await;
+ for _ in 0..2 {
+ reservations.insert(
+ AgentSessionId::new(),
+ (owner.clone(), std::time::Instant::now()),
+ );
+ }
+ }
+ assert!(
+ service
+ .warm_session(owner, AgentSessionId::new())
+ .await
+ .unwrap()
+ .is_none()
+ );
+
+ let attributes = attributes(&exporter, &provider, "agent.session.prepare_warm");
+ assert_eq!(
+ attributes.get("agent.warm.outcome"),
+ Some(&opentelemetry::Value::from("refused_owner_cap"))
+ );
+ assert!(attributes.contains_key("agent.session.model"));
+}
diff --git a/crates/agent_harness/src/domain/service/warm.rs b/crates/agent_harness/src/domain/service/warm.rs
index 0c521020f9..8b2c4cb1e4 100644
--- a/crates/agent_harness/src/domain/service/warm.rs
+++ b/crates/agent_harness/src/domain/service/warm.rs
@@ -4,6 +4,7 @@ use super::*;
use crate::domain::model::SessionRepository;
use agent_session::domain::model::session_owner_user;
use agent_session::domain::ports::{OpenManagedSession, SelectedManagedPersona};
+use agent_session::domain::warm::WarmClaim;
use model_owner::Owner;
use std::time::{Duration, Instant};
@@ -51,7 +52,7 @@ where
Mentions: PromptMentions,
Notifier: AgentSessionNotifier,
{
- #[tracing::instrument(skip_all, err, fields(agent.session.id = tracing::field::Empty, agent.harness = tracing::field::Empty, agent.session.warm = warm, agent.session.warm_hit = false))]
+ #[tracing::instrument(skip_all, err, fields(agent.session.id = tracing::field::Empty, agent.harness = tracing::field::Empty, agent.session.warm = warm, agent.session.warm_hit = false, agent.session.warm_id_sent = tracing::field::Empty, agent.session.warm_miss_reason = tracing::field::Empty, agent.session.model = tracing::field::Empty))]
pub(super) async fn open_managed_session_inner(
&self,
request: agent_session::domain::ports::OpenManagedSession,
@@ -96,6 +97,7 @@ where
// in is choosing what this session runs on, for its whole life.
let requested_model = request.model;
let model = requested_model.clone().unwrap_or(model);
+ tracing::Span::current().record("agent.session.model", &model);
let kind = AgentKind::for_session(bot_id, &harness);
let harness = kind.harness_slug().map_or(harness, str::to_owned);
tracing::Span::current().record("agent.harness", &harness);
@@ -154,22 +156,31 @@ where
.instrument(tracing::info_span!("agent.init.admission", agent.session.id = %session_id))
.await
.map_err(into_session_error)?;
- if !warm
- && kind == AgentKind::InMemory
- && bot_id == bot_id::MACRO_NEW_BOT_ID
- && matches!(mcp_servers, AgentMcpServers::OwnerConnections)
- && request.repo_url.is_none()
- && let Some(lifecycle) = &self.warm_lifecycle
- && lifecycle
- .claim(
- session_id,
- &request.owner,
- bot_id,
- &model,
- instructions.as_deref(),
- )
- .await?
- {
+ let claim = if warm {
+ None
+ } else {
+ let eligible = kind == AgentKind::InMemory
+ && bot_id == bot_id::MACRO_NEW_BOT_ID
+ && matches!(mcp_servers, AgentMcpServers::OwnerConnections)
+ && request.repo_url.is_none();
+ let claim = match &self.warm_lifecycle {
+ Some(lifecycle) if eligible => Some(
+ lifecycle
+ .claim(
+ session_id,
+ &request.owner,
+ bot_id,
+ &model,
+ instructions.as_deref(),
+ )
+ .await?,
+ ),
+ _ => None,
+ };
+ record_warm_claim(request.id.is_some(), eligible, claim);
+ claim
+ };
+ if claim == Some(WarmClaim::Claimed) {
self.warm_reservations.lock().await.remove(&session_id);
tracing::Span::current().record("agent.session.warm_hit", true);
let session = self.inner.sessions.get_session(session_id).await?;
@@ -357,28 +368,40 @@ where
self
}
+ #[tracing::instrument(name = "agent.session.prepare_warm", skip_all, fields(agent.session.id = %id, agent.warm.outcome = tracing::field::Empty, agent.session.model = tracing::field::Empty))]
pub(super) async fn prepare_warm(
&self,
owner: Owner,
id: AgentSessionId,
) -> agent_session::domain::error::Result<Option<AgentSession>> {
+ let span = tracing::Span::current();
session_owner_user(&owner)?;
+ span.record(
+ "agent.session.model",
+ &self.inner.defaults.for_bot(bot_id::MACRO_NEW_BOT_ID).model,
+ );
if self.warm_lifecycle.is_none() {
+ span.record("agent.warm.outcome", "disabled");
return Ok(None);
}
{
let mut reservations = self.warm_reservations.lock().await;
reservations.retain(|_, (_, started)| started.elapsed() < WARM_RESERVATION_TTL);
if reservations.contains_key(&id) {
+ span.record("agent.warm.outcome", "reused");
+ return Ok(None);
+ }
+ if reservations.len() >= MAX_WARM_RESERVATIONS {
+ span.record("agent.warm.outcome", "refused_global_cap");
return Ok(None);
}
- if reservations.len() >= MAX_WARM_RESERVATIONS
- || reservations
- .values()
- .filter(|(user, _)| user == &owner)
- .count()
- >= MAX_WARM_RESERVATIONS_PER_OWNER
+ if reservations
+ .values()
+ .filter(|(user, _)| user == &owner)
+ .count()
+ >= MAX_WARM_RESERVATIONS_PER_OWNER
{
+ span.record("agent.warm.outcome", "refused_owner_cap");
return Ok(None);
}
reservations.insert(id, (owner.clone(), Instant::now()));
@@ -402,12 +425,17 @@ where
)
.await
{
- Ok(Ok(session)) => Ok(Some(session)),
+ Ok(Ok(session)) => {
+ span.record("agent.warm.outcome", "prepared");
+ Ok(Some(session))
+ }
Ok(Err(error)) => {
+ span.record("agent.warm.outcome", "failed");
tracing::warn!(error = ?error, "warm session preparation failed");
Ok(None)
}
Err(_) => {
+ span.record("agent.warm.outcome", "timed_out");
tracing::warn!("warm session preparation timed out");
Ok(None)
}
@@ -432,3 +460,21 @@ where
}
}
}
+
+/// Why a non-warming open did or did not claim a warm session, on its span.
+fn record_warm_claim(client_minted_id: bool, eligible: bool, claim: Option<WarmClaim>) {
+ let span = tracing::Span::current();
+ let reason: &'static str = match claim {
+ _ if !client_minted_id => "no_warm_id",
+ Some(WarmClaim::Claimed) => "none",
+ Some(WarmClaim::Missed(reason)) => reason.into(),
+ None if !eligible => "ineligible",
+ None => "disabled",
+ };
+ span.record("agent.session.warm_miss_reason", reason);
+ if !client_minted_id {
+ span.record("agent.session.warm_id_sent", false);
+ } else if let Some(claim) = claim {
+ span.record("agent.session.warm_id_sent", claim.found_warm_session());
+ }
+}
diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rs
index 9c1ea771cb..242fcca82f 100644
--- a/crates/agent_session/src/domain/service.rs
+++ b/crates/agent_session/src/domain/service.rs
@@ -592,6 +592,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
agent.action.name = action.as_ref(),
agent.command.queue_wait_ms = tracing::field::Empty,
agent.session.runtime_phase_at_dequeue = tracing::field::Empty,
+ agent.command.handshake_wait_ms = tracing::field::Empty,
otel.status_code = tracing::field::Empty,
otel.status_description = tracing::field::Empty,
);
diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rs
index e40c6bb733..c5443593f1 100644
--- a/crates/agent_session/src/domain/service/test.rs
+++ b/crates/agent_session/src/domain/service/test.rs
@@ -2161,6 +2161,157 @@ async fn a_prompt_turn_is_traced_as_an_agent_span_under_its_command() {
assert!(input.contains("what time is it?"), "{input}");
}
+/// A command dequeued before the handshake finished says how long it waited
+/// for it; one dequeued on a live session waited for nothing.
+#[tokio::test]
+async fn a_command_span_records_its_wait_for_the_handshake() {
+ use opentelemetry::trace::TracerProvider as _;
+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};
+ use tracing_subscriber::layer::SubscriberExt as _;
+
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));
+ let _guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ tracing::callsite::rebuild_interest_cache();
+
+ let repo = InMemoryAgentSessionRepo::new();
+ let session = test_session();
+ repo.insert_session(test_agent_session(session));
+ let entered = Arc::new(Notify::new());
+ let release = Arc::new(Notify::new());
+ let logs = BlockingPromptLogs {
+ repo: repo.clone(),
+ entered: entered.clone(),
+ release: release.clone(),
+ hang_disconnect: false,
+ fail_restore_log: None,
+ };
+ let (outbound_tx, mut outbound_rx) = mpsc::channel(8);
+ let (inbound_tx, inbound_rx) = mpsc::channel(8);
+ let (commands, command_rx) = mpsc::channel(8);
+ let (handshake, _) = watch::channel(HandshakeStatus::Pending);
+ let actor = SessionActor::new(
+ session,
+ None,
+ "/workspace".to_owned(),
+ Vec::new(),
+ crate::domain::session::PermissionPolicy::AutoAccept,
+ RecordingTransport {
+ outbound: outbound_tx,
+ inbound: inbound_rx,
+ },
+ logs,
+ command_rx,
+ handshake,
+ Arc::new(crate::domain::ports::NoOpTurnObserver),
+ Arc::new(crate::domain::ports::NoOpToolCatalog),
+ );
+ let active = Arc::new(ActiveSessions::new());
+ let (stopped_tx, _) = watch::channel(false);
+ let claim = claim_for_test(&repo, session).await;
+ let task = tokio::spawn(
+ run_session(
+ actor,
+ Arc::downgrade(&active),
+ Arc::new(()),
+ stopped_tx,
+ CancellationToken::new(),
+ repo.clone(),
+ claim,
+ Arc::new(crate::domain::ports::NoOpTurnObserver),
+ )
+ .with_current_subscriber(),
+ );
+ let send_prompt = |name: &'static str| {
+ let (completed, result) = oneshot::channel();
+ let command = SessionCommand {
+ user_id: None,
+ action: AgentAction::prompt(name),
+ action_id: AgentActionId::mint(),
+ completed,
+ span: tracing::info_span!(
+ "agent.session.command",
+ agent.command.name = name,
+ agent.command.queue_wait_ms = tracing::field::Empty,
+ agent.session.runtime_phase_at_dequeue = tracing::field::Empty,
+ agent.command.handshake_wait_ms = tracing::field::Empty,
+ ),
+ enqueued_at: tokio::time::Instant::now(),
+ };
+ (command, result)
+ };
+
+ // Queued before the runtime is even ready: it waits out the handshake.
+ let (early, early_result) = send_prompt("early");
+ commands.send(early).await.unwrap();
+ open_test_session(&inbound_tx, &mut outbound_rx, session).await;
+ entered.notified().await;
+ release.notify_one();
+ outbound_rx
+ .recv()
+ .await
+ .expect("early prompt is dispatched");
+ early_result
+ .await
+ .unwrap()
+ .expect("early delivery completes");
+
+ let (live, live_result) = send_prompt("live");
+ commands.send(live).await.unwrap();
+ entered.notified().await;
+ release.notify_one();
+ outbound_rx.recv().await.expect("live prompt is dispatched");
+ live_result.await.unwrap().expect("live delivery completes");
+
+ drop(inbound_tx);
+ task.await.unwrap();
+ provider.force_flush().expect("flush");
+ let spans = exporter.get_finished_spans().expect("finished spans");
+ let attribute = |name: &str, key: &str| {
+ let span = spans
+ .iter()
+ .find(|span| {
+ span.name == "agent.session.command"
+ && span.attributes.iter().any(|kv| {
+ kv.key.as_str() == "agent.command.name" && kv.value.as_str() == name
+ })
+ })
+ .unwrap_or_else(|| panic!("no {name} command span: {spans:#?}"));
+ span.attributes
+ .iter()
+ .find(|kv| kv.key.as_str() == key)
+ .map(|kv| kv.value.clone())
+ };
+ assert!(matches!(
+ attribute("early", "agent.session.runtime_phase_at_dequeue")
+ .expect("phase recorded")
+ .as_str()
+ .as_ref(),
+ "booting" | "handshaking"
+ ));
+ assert!(matches!(
+ attribute("early", "agent.command.handshake_wait_ms"),
+ Some(opentelemetry::Value::I64(wait)) if wait >= 0
+ ));
+ assert_eq!(
+ attribute("live", "agent.session.runtime_phase_at_dequeue")
+ .expect("phase recorded")
+ .as_str(),
+ "live"
+ );
+ assert_eq!(
+ attribute("live", "agent.command.handshake_wait_ms"),
+ Some(opentelemetry::Value::I64(0))
+ );
+ assert!(matches!(
+ attribute("live", "agent.command.queue_wait_ms"),
+ Some(opentelemetry::Value::I64(_))
+ ));
+}
+
mod initial_model;
mod owner_binding;
mod turn_projection;
diff --git a/crates/agent_session/src/domain/session/actors.rs b/crates/agent_session/src/domain/session/actors.rs
index a37dbfb938..1924fb3935 100644
--- a/crates/agent_session/src/domain/session/actors.rs
+++ b/crates/agent_session/src/domain/session/actors.rs
@@ -62,6 +62,8 @@ pub(crate) struct SessionCommand {
pub(crate) struct SessionCompletion {
completed: oneshot::Sender<Result<()>>,
span: tracing::Span,
+ /// When the command was dequeued, if the runtime was not live yet.
+ dequeued_before_live: Option<Instant>,
}
/// Whether a [`SessionActor`] has more steps to take.
@@ -238,17 +240,17 @@ where
Some(SessionCommand { user_id, action, action_id, completed, span, enqueued_at }) => {
span.record(
"agent.command.queue_wait_ms",
- enqueued_at.elapsed().as_millis() as u64,
- );
- span.record(
- "agent.session.runtime_phase_at_dequeue",
- self.machine.status().as_ref(),
+ elapsed_ms(enqueued_at),
);
+ let status = self.machine.status();
+ span.record("agent.session.runtime_phase_at_dequeue", status.as_ref());
+ let dequeued_before_live = (!matches!(status, RuntimeStatus::Live { .. }))
+ .then(Instant::now);
Input::Command {
from: user_id,
action,
action_id,
- token: SessionCompletion { completed, span },
+ token: SessionCompletion { completed, span, dequeued_before_live },
}
},
// The service dropped every handle; nobody can reach us.
@@ -301,7 +303,13 @@ where
match effect {
Effect::Send { from, mut message } => {
let command_span = effects.front().and_then(|effect| match effect {
- Effect::Complete { token, .. } => Some(token.span.clone()),
+ Effect::Complete { token, .. } => {
+ token.span.record(
+ "agent.command.handshake_wait_ms",
+ token.dequeued_before_live.map_or(0, elapsed_ms),
+ );
+ Some(token.span.clone())
+ }
_ => None,
});
// Before delivery, so the turn's span starts when the
@@ -732,6 +740,11 @@ where
}
}
+/// Milliseconds as i64: a u64 reaches OTel as a string.
+fn elapsed_ms(since: Instant) -> i64 {
+ i64::try_from(since.elapsed().as_millis()).unwrap_or(i64::MAX)
+}
+
fn acp_method(message: &ToRuntimeMessage) -> Option<&str> {
let ToRuntimeMessage::Acp(AcpMessage(frame)) = message else {
return None;
diff --git a/crates/agent_session/src/domain/warm.rs b/crates/agent_session/src/domain/warm.rs
index ae5901e89a..5bbc00af62 100644
--- a/crates/agent_session/src/domain/warm.rs
+++ b/crates/agent_session/src/domain/warm.rs
@@ -6,6 +6,41 @@ use model_owner::Owner;
use std::future::Future;
use std::pin::Pin;
+/// What a claim found; a miss says the first claim condition that failed.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum WarmClaim {
+ Claimed,
+ Missed(WarmMissReason),
+}
+
+/// Why a claim missed, in the order the claim checks them.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)]
+#[strum(serialize_all = "snake_case")]
+pub enum WarmMissReason {
+ /// No session has this id: the client sent a fresh id, or the warm row is gone.
+ NotFound,
+ /// The session is visible or belongs to a thread.
+ NotWarm,
+ OwnerMismatch,
+ BotMismatch,
+ ModelMismatch,
+ InstructionsMismatch,
+ Disconnected,
+ Expired,
+ /// Every condition held, but a concurrent claim promoted it first.
+ LostRace,
+}
+
+impl WarmClaim {
+ /// Whether the id named a warm session at all.
+ pub fn found_warm_session(self) -> bool {
+ !matches!(
+ self,
+ Self::Missed(WarmMissReason::NotFound | WarmMissReason::NotWarm)
+ )
+ }
+}
+
/// Persistence operations whose claim and expiry must serialize across replicas.
pub trait WarmSessionLifecycle: Send + Sync {
/// Reveal only a fresh warm session belonging to this owner and exact persona.
@@ -16,7 +51,7 @@ pub trait WarmSessionLifecycle: Send + Sync {
bot: BotId,
model: &'a str,
instructions: Option<&'a str>,
- ) -> Pin<Box<dyn Future<Output = Result<bool>> + Send + 'a>>;
+ ) -> Pin<Box<dyn Future<Output = Result<WarmClaim>> + Send + 'a>>;
/// Mark expired hidden sessions disconnected, returning ids for routed deletion.
/// Claimed sessions are never selected. Work is bounded per sweep.
fn expire(&self) -> Pin<Box<dyn Future<Output = Result<Vec<AgentSessionId>>> + Send + '_>>;
diff --git a/crates/agent_session/src/outbound/postgres/test.rs b/crates/agent_session/src/outbound/postgres/test.rs
index de15e7adbb..6c38be1445 100644
--- a/crates/agent_session/src/outbound/postgres/test.rs
+++ b/crates/agent_session/src/outbound/postgres/test.rs
@@ -3009,7 +3009,7 @@ async fn the_originating_channel_steers_only_shared_agents_sessions(pool: PgPool
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPool) {
- use crate::domain::warm::WarmSessionLifecycle;
+ use crate::domain::warm::{WarmClaim, WarmMissReason, WarmSessionLifecycle};
let bot = create_test_bot(&pool).await;
let repo = test_repo(&pool);
let mut params = new_session(bot, None, None);
@@ -3024,13 +3024,14 @@ async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPoo
.is_empty()
);
let other = Owner::User(user_id("macro|[email protected]"));
- assert!(
- !WarmSessionLifecycle::claim(&repo, session.id, &other, bot, &session.model, None)
+ assert_eq!(
+ WarmSessionLifecycle::claim(&repo, session.id, &other, bot, &session.model, None)
.await
- .unwrap()
+ .unwrap(),
+ WarmClaim::Missed(WarmMissReason::OwnerMismatch)
);
- assert!(
- !WarmSessionLifecycle::claim(
+ assert_eq!(
+ WarmSessionLifecycle::claim(
&repo,
session.id,
&session.owner_id,
@@ -3039,10 +3040,11 @@ async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPoo
None
)
.await
- .unwrap()
+ .unwrap(),
+ WarmClaim::Missed(WarmMissReason::ModelMismatch)
);
- assert!(
- !WarmSessionLifecycle::claim(
+ assert_eq!(
+ WarmSessionLifecycle::claim(
&repo,
session.id,
&session.owner_id,
@@ -3051,7 +3053,21 @@ async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPoo
Some("different instructions")
)
.await
- .unwrap()
+ .unwrap(),
+ WarmClaim::Missed(WarmMissReason::InstructionsMismatch)
+ );
+ assert_eq!(
+ WarmSessionLifecycle::claim(
+ &repo,
+ AgentSessionId::new(),
+ &session.owner_id,
+ bot,
+ &session.model,
+ None
+ )
+ .await
+ .unwrap(),
+ WarmClaim::Missed(WarmMissReason::NotFound)
);
let (left, right) = tokio::join!(
WarmSessionLifecycle::claim(
@@ -3071,7 +3087,14 @@ async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPoo
None
),
);
- assert_ne!(left.unwrap(), right.unwrap());
+ let mut outcomes = [left.unwrap(), right.unwrap()];
+ outcomes.sort_by_key(|claim| *claim != WarmClaim::Claimed);
+ // The loser either waited on the winner's row lock or read the claimed row.
+ assert_eq!(outcomes[0], WarmClaim::Claimed);
+ assert!(matches!(
+ outcomes[1],
+ WarmClaim::Missed(WarmMissReason::LostRace | WarmMissReason::NotWarm)
+ ));
assert_eq!(
repo.recent_for_owner(&user_id(OWNER), NonZeroUsize::new(10).unwrap())
.await
@@ -3084,7 +3107,7 @@ async fn warm_sessions_are_hidden_and_claimed_only_once_by_the_owner(pool: PgPoo
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
async fn expired_warm_sessions_cannot_be_claimed(pool: PgPool) {
- use crate::domain::warm::WarmSessionLifecycle;
+ use crate::domain::warm::{WarmClaim, WarmMissReason, WarmSessionLifecycle};
let bot = create_test_bot(&pool).await;
let repo = test_repo(&pool);
let mut params = new_session(bot, None, None);
@@ -3097,9 +3120,22 @@ async fn expired_warm_sessions_cannot_be_claimed(pool: PgPool) {
.execute(&pool)
.await
.unwrap();
+ assert_eq!(
+ WarmSessionLifecycle::claim(
+ &repo,
+ session.id,
+ &session.owner_id,
+ bot,
+ &session.model,
+ None
+ )
+ .await
+ .unwrap(),
+ WarmClaim::Missed(WarmMissReason::Expired)
+ );
assert_eq!(repo.expire().await.unwrap(), vec![session.id]);
- assert!(
- !WarmSessionLifecycle::claim(
+ assert_eq!(
+ WarmSessionLifecycle::claim(
&repo,
session.id,
&session.owner_id,
@@ -3108,7 +3144,8 @@ async fn expired_warm_sessions_cannot_be_claimed(pool: PgPool) {
None
)
.await
- .unwrap()
+ .unwrap(),
+ WarmClaim::Missed(WarmMissReason::Disconnected)
);
AgentSessionRepo::delete(&repo, session.id).await.unwrap();
assert!(repo.expire().await.unwrap().is_empty());
diff --git a/crates/agent_session/src/outbound/postgres/warm.rs b/crates/agent_session/src/outbound/postgres/warm.rs
index 3e891fb8f9..82613d31df 100644
--- a/crates/agent_session/src/outbound/postgres/warm.rs
+++ b/crates/agent_session/src/outbound/postgres/warm.rs
@@ -1,6 +1,6 @@
//! Warm sessions use list_hidden without a thread; hidden mentions always have a thread.
use super::*;
-use crate::domain::warm::WarmSessionLifecycle;
+use crate::domain::warm::{WarmClaim, WarmMissReason, WarmSessionLifecycle};
use std::{future::Future, pin::Pin};
impl<B: BotFacts + 'static> WarmSessionLifecycle for PgAgentSessionRepo<B> {
@@ -11,19 +11,46 @@ impl<B: BotFacts + 'static> WarmSessionLifecycle for PgAgentSessionRepo<B> {
bot: BotId,
model: &'a str,
instructions: Option<&'a str>,
- ) -> Pin<Box<dyn Future<Output = Result<bool>> + Send + 'a>> {
+ ) -> Pin<Box<dyn Future<Output = Result<WarmClaim>> + Send + 'a>> {
Box::pin(async move {
let user = session_owner_user(owner)?;
let mut transaction = self.pool.begin().await.context("begin warm claim")?;
- let changed = sqlx::query!(
+ // One statement: the snapshot that explains a miss is the one the claim ran against.
+ let row = sqlx::query!(
r#"
- UPDATE agent_session SET list_hidden = false
- WHERE id = $1 AND owner_id = $2 AND bot_id = $3 AND model = $4
- AND instructions IS NOT DISTINCT FROM $5
- AND list_hidden AND thread_id IS NULL
- AND status != 'disconnected'
- AND NOT (status = 'event' AND status_event_name = 'disconnected')
- AND created_at > NOW() - INTERVAL '10 minutes'
+ WITH candidate AS (
+ SELECT
+ true AS found,
+ list_hidden AND thread_id IS NULL AS warm,
+ owner_id = $2 AS owner_matches,
+ bot_id = $3 AS bot_matches,
+ model = $4 AS model_matches,
+ instructions IS NOT DISTINCT FROM $5 AS instructions_match,
+ COALESCE(status != 'disconnected'
+ AND NOT (status = 'event' AND status_event_name = 'disconnected'), false) AS live,
+ created_at > NOW() - INTERVAL '10 minutes' AS fresh
+ FROM agent_session WHERE id = $1
+ ), claimed AS (
+ UPDATE agent_session SET list_hidden = false
+ WHERE id = $1 AND owner_id = $2 AND bot_id = $3 AND model = $4
+ AND instructions IS NOT DISTINCT FROM $5
+ AND list_hidden AND thread_id IS NULL
+ AND status != 'disconnected'
+ AND NOT (status = 'event' AND status_event_name = 'disconnected')
+ AND created_at > NOW() - INTERVAL '10 minutes'
+ RETURNING id
+ )
+ SELECT
+ EXISTS (SELECT 1 FROM claimed) AS "claimed!",
+ candidate.found AS "found?",
+ candidate.warm AS "warm?",
+ candidate.owner_matches AS "owner_matches?",
+ candidate.bot_matches AS "bot_matches?",
+ candidate.model_matches AS "model_matches?",
+ candidate.instructions_match AS "instructions_match?",
+ candidate.live AS "live?",
+ candidate.fresh AS "fresh?"
+ FROM (SELECT 1) AS one LEFT JOIN candidate ON true
"#,
id.as_uuid(),
user.as_ref(),
@@ -31,18 +58,37 @@ impl<B: BotFacts + 'static> WarmSessionLifecycle for PgAgentSessionRepo<B> {
model,
instructions
)
- .execute(&mut *transaction)
+ .fetch_one(&mut *transaction)
.await
- .context("claim warm session")?
- .rows_affected()
- == 1;
- if changed {
+ .context("claim warm session")?;
+ let checks = [
+ (row.warm, WarmMissReason::NotWarm),
+ (row.owner_matches, WarmMissReason::OwnerMismatch),
+ (row.bot_matches, WarmMissReason::BotMismatch),
+ (row.model_matches, WarmMissReason::ModelMismatch),
+ (row.instructions_match, WarmMissReason::InstructionsMismatch),
+ (row.live, WarmMissReason::Disconnected),
+ (row.fresh, WarmMissReason::Expired),
+ ];
+ let claim = if row.claimed {
+ WarmClaim::Claimed
+ } else if row.found.is_none() {
+ WarmClaim::Missed(WarmMissReason::NotFound)
+ } else {
+ WarmClaim::Missed(
+ checks
+ .into_iter()
+ .find_map(|(held, reason)| (held != Some(true)).then_some(reason))
+ .unwrap_or(WarmMissReason::LostRace),
+ )
+ };
+ if claim == WarmClaim::Claimed {
upsert_user_history(&mut transaction, user.as_ref(), &id.as_uuid())
.await
.context("record claimed warm session history")?;
}
transaction.commit().await.context("commit warm claim")?;
- Ok(changed)
+ Ok(claim)
})
}
fn expire(&self) -> Pin<Box<dyn Future<Output = Result<Vec<AgentSessionId>>> + Send + '_>> {
diff --git a/deny.toml b/deny.toml
index ccf1225603..5217537605 100644
--- a/deny.toml
+++ b/deny.toml
@@ -201,6 +201,7 @@ deny = [
], reason = "Use macro_aws_config for AWS configuration in workspace crates" },
{ crate = "tracing-subscriber", wrappers = [
"agent", # GenAI span tests
+ "agent_harness", # Warm-claim and handshake span tests
"agent_inmem", # GenAI trace propagation tests
"agent_session", # GenAI span tests
"ai_toolset", # GenAI span tests