Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(otel): trace agent triggers end to end (#5819)

Wolf Mermelstein87788136afeb
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 89d34bd2b7..adebd361e5 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -94,6 +94,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -180,6 +181,7 @@
"crates/macro_event_topics",
"crates/macro_queues",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -255,6 +257,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -314,6 +317,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -377,6 +381,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -947,6 +952,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -1032,6 +1038,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/maybe_send",
@@ -1136,6 +1143,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -1326,6 +1334,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -1519,6 +1528,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -1671,6 +1681,7 @@
"crates/macro_authorization",
"crates/macro_env",
"crates/macro_env_var",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/model",
@@ -1710,6 +1721,7 @@
"crates/macro_entrypoint",
"crates/macro_env",
"crates/macro_env_var",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/model",
@@ -1732,6 +1744,7 @@
"crates/connection_gateway_client",
"crates/connection_gateway_models",
"crates/cowlike",
+ "crates/macro_tower_layers",
"crates/model-entity",
"crates/workspace-hack"
],
@@ -2970,6 +2983,7 @@
"crates/macro_middleware",
"crates/macro_queues",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -3518,6 +3532,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -4087,6 +4102,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mcp_client",
@@ -4223,6 +4239,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -4300,6 +4317,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -5066,6 +5084,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -5147,6 +5166,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -5211,6 +5231,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -5432,6 +5453,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mcp_client",
@@ -6197,6 +6219,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -6278,6 +6301,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -6401,6 +6425,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -6512,6 +6537,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -6596,6 +6622,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -6671,6 +6698,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/maybe_send",
@@ -6744,6 +6772,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -7109,6 +7138,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -7185,6 +7215,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -7275,6 +7306,7 @@
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_tower_layers",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
diff --git a/Cargo.lock b/Cargo.lock
index 5aba238791..440b291378 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -217,6 +217,7 @@ dependencies = [
"macro_authorization",
"macro_env_var",
"macro_event_broker",
+ "macro_tower_layers",
"macro_user_id",
"macro_uuid",
"mockall",
@@ -297,6 +298,7 @@ dependencies = [
"specta",
"specta-serde",
"specta-typescript",
+ "strum 0.27.2",
"thiserror 2.0.18",
"tokio",
"tokio-tungstenite 0.28.0",
@@ -339,6 +341,7 @@ dependencies = [
"thiserror 2.0.18",
"tokio",
"tokio-tungstenite 0.28.0",
+ "tokio-util",
"tower 0.5.3",
"tracing",
"utoipa",
@@ -3648,11 +3651,13 @@ dependencies = [
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_tower_layers",
"macro_user_id",
"mockall",
"model",
"model-entity",
"models_bulk_upload",
+ "opentelemetry",
"redis",
"secretsmanager_client",
"serde",
@@ -3666,6 +3671,7 @@ dependencies = [
"tower 0.5.3",
"tower-http",
"tracing",
+ "tracing-opentelemetry",
"tungstenite 0.28.0",
"url",
"utoipa",
@@ -3680,6 +3686,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"connection_gateway_models",
+ "macro_tower_layers",
"model-entity",
"reqwest 0.13.4",
"serde",
@@ -8555,10 +8562,12 @@ dependencies = [
"either",
"macro_env",
"macro_env_var",
+ "opentelemetry",
"rdkafka",
"thiserror 2.0.18",
"tokio",
"tracing",
+ "tracing-opentelemetry",
"uuid",
"workspace-hack",
]
diff --git a/crates/agent_harness/Cargo.toml b/crates/agent_harness/Cargo.toml
index 3c682582b1..c8ae2eaefc 100644
--- a/crates/agent_harness/Cargo.toml
+++ b/crates/agent_harness/Cargo.toml
@@ -23,6 +23,7 @@ kafka_util = { path = "../kafka_util" }
macro_env_var = { path = "../macro_env_var" }
macro_event_broker = { path = "../macro_event_broker", features = ["outbound"] }
macro_authorization = { path = "../macro_authorization", features = ["axum"] }
+macro_tower_layers = { path = "../macro_tower_layers" }
macro_user_id = { path = "../macro_user_id" }
macro_uuid = { path = "../macro_uuid" }
reqwest = { workspace = true }
diff --git a/crates/agent_harness/src/domain/ports/test.rs b/crates/agent_harness/src/domain/ports/test.rs
index fd1d31effc..49148dd47b 100644
--- a/crates/agent_harness/src/domain/ports/test.rs
+++ b/crates/agent_harness/src/domain/ports/test.rs
@@ -99,6 +99,7 @@ async fn container_session_runs_and_logs_end_to_end() {
agent.wait_for_requests(2).await;
agent.opens_session(NewSessionResponse::new("acp-container-test"));
agent.wait_for_requests(3).await;
+ agent.completes_prompt().await;
send.await.unwrap().unwrap();
assert_eq!(
@@ -116,15 +117,14 @@ async fn container_session_runs_and_logs_end_to_end() {
assert!(matches!(logs[5].entry.content, Message::ToRuntime(_)));
assert_eq!(logs[5].entry.user_id, Some(owner()));
- sessions
- .send_action(
- id,
- Some(owner()),
- AgentAction::prompt("and run clippy"),
- AgentActionId::mint(),
- )
- .await
- .unwrap();
+ let send = sessions.send_action(
+ id,
+ Some(owner()),
+ AgentAction::prompt("and run clippy"),
+ AgentActionId::mint(),
+ );
+ let (result, ()) = tokio::join!(send, agent.completes_prompt());
+ result.unwrap();
assert_eq!(containers.spawned(), 1);
assert_eq!(containers.resumed(), 0);
assert_eq!(
diff --git a/crates/agent_harness/src/domain/service.rs b/crates/agent_harness/src/domain/service.rs
index 9d32d84a55..cf4be0e772 100644
--- a/crates/agent_harness/src/domain/service.rs
+++ b/crates/agent_harness/src/domain/service.rs
@@ -16,6 +16,8 @@ use dashmap::DashMap;
use dashmap::mapref::entry::Entry;
use macro_user_id::user_id::MacroUserIdStr;
use tokio::sync::{mpsc, oneshot};
+use tracing::Instrument as _;
+use tracing::instrument::WithSubscriber as _;
use crate::domain::error::{HarnessError, Result};
use crate::domain::model::{
@@ -30,6 +32,9 @@ type SessionWorkers = DashMap<AgentSessionId, mpsc::UnboundedSender<QueuedComman
struct QueuedCommand {
command: HarnessCommand,
completed: oneshot::Sender<Result<()>>,
+ /// The caller's span, carried across the queue so the work the worker does
+ /// on its own task still hangs off whatever triggered it.
+ span: tracing::Span,
}
struct AgentHarnessInner<Sessions, Containers, Announcer, Runtimes> {
@@ -77,16 +82,23 @@ where
/// Queue one command behind any work already running for its session.
///
/// Queue admission happens synchronously so callers can spawn the returned
- /// completion future without reordering commands.
+ /// completion future without reordering commands. It is also where the
+ /// caller's span is captured: the returned future only awaits a oneshot, so
+ /// the span has to travel with the command to reach the work itself.
pub fn execute(
&self,
session_id: AgentSessionId,
mut command: HarnessCommand,
) -> impl Future<Output = Result<()>> + Send + 'static {
+ let caller = tracing::Span::current();
let result = loop {
let commands = self.commands(session_id);
let (completed, result) = oneshot::channel();
- let queued = QueuedCommand { command, completed };
+ let queued = QueuedCommand {
+ command,
+ completed,
+ span: caller.clone(),
+ };
match commands.send(queued) {
Ok(()) => break result,
@@ -122,7 +134,10 @@ where
session_id: AgentSessionId,
receiver: mpsc::UnboundedReceiver<QueuedCommand>,
) {
- tokio::spawn(run_session_worker(session_id, self.inner.clone(), receiver));
+ // The worker outlives the call that created it, so it has to carry the
+ // subscriber forward itself or every command it runs traces nowhere.
+ let inner = self.inner.clone();
+ tokio::spawn(run_session_worker(session_id, inner, receiver).with_current_subscriber());
}
/// Post the announcement for a prompt an external runtime delivers.
@@ -458,9 +473,14 @@ where
%session_id,
bot_id = %command.bot_id,
message_id = %command.origin.message_id,
+ channel_id = %command.origin.channel_id,
+ thread_id = %command.origin.thread_id,
+ agent.trigger.kind = "mention",
+ agent.session.id = tracing::field::Empty,
))]
async fn open(&self, session_id: AgentSessionId, command: OpenSession) -> Result<()> {
let OpenSession { bot_id, origin } = command;
+ tracing::Span::current().record("agent.session.id", tracing::field::display(session_id));
let repo_url = self.defaults.repo_url.clone();
let sandbox_size = self.sessions.user_sandbox_size(&origin.sender).await?;
@@ -537,7 +557,7 @@ where
/// Three steps, in this order: persist whatever the action changes about
/// the session, work out whether anyone needs telling, then deliver it.
/// Announcing last means nothing is announced that was never sent.
- #[tracing::instrument(err, skip(self, command), fields(%session_id))]
+ #[tracing::instrument(err, skip(self, command), fields(agent.session.id = %session_id))]
async fn deliver(&self, session_id: AgentSessionId, command: DeliverAction) -> Result<()> {
let DeliverAction {
id,
@@ -641,8 +661,12 @@ async fn run_session_worker<Sessions, Containers, Announcer, Runtimes>(
Runtimes: RuntimeConnections,
{
while let Some(queued) = receiver.recv().await {
- let QueuedCommand { command, completed } = queued;
- let result = inner.execute(session_id, command).await;
+ let QueuedCommand {
+ command,
+ completed,
+ span,
+ } = queued;
+ let result = inner.execute(session_id, command).instrument(span).await;
let _ = completed.send(result);
}
}
diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rs
index 0076bf994c..22d4ecb9ab 100644
--- a/crates/agent_harness/src/domain/service/test.rs
+++ b/crates/agent_harness/src/domain/service/test.rs
@@ -119,7 +119,7 @@ fn harness() -> (
}
/// Play the agent's half of the ACP handshake.
-async fn complete_handshake(container: &ContainerMock) {
+async fn complete_session_handshake(container: &ContainerMock) {
let agent = container.agent();
let already = agent.received_requests().len();
container.sends_ready();
@@ -129,6 +129,12 @@ async fn complete_handshake(container: &ContainerMock) {
agent.opens_session(NewSessionResponse::new("acp-test"));
}
+async fn complete_handshake(container: &ContainerMock) {
+ complete_session_handshake(container).await;
+ let agent = container.agent();
+ agent.completes_prompt().await;
+}
+
async fn complete_resume(container: &ContainerMock) {
let agent = container.agent();
container.sends_ready();
@@ -146,6 +152,7 @@ async fn complete_resume(container: &ContainerMock) {
ClientRequest::ResumeSessionRequest(request) if request.session_id.to_string() == "acp-test"
));
agent.resumes_session(ResumeSessionResponse::new());
+ agent.completes_prompt().await;
}
fn prompts(agent: &FakeAgent) -> Vec<Vec<ContentBlock>> {
@@ -333,13 +340,13 @@ async fn forward_to_a_live_session_reuses_the_transport() {
let (opened, container) = tokio::join!(open, drive);
opened.expect("open should succeed");
- service
- .execute(
- id,
- HarnessCommand::Deliver(forward_message("and add a regression test")),
- )
- .await
- .expect("forward to a live session should succeed");
+ let forward = service.execute(
+ id,
+ HarnessCommand::Deliver(forward_message("and add a regression test")),
+ );
+ let agent = container.agent();
+ let (result, ()) = tokio::join!(forward, agent.completes_prompt());
+ result.expect("forward to a live session should succeed");
assert_eq!(containers.spawned(), 1, "no second container");
assert_eq!(containers.resumed(), 0, "no resume for a live session");
@@ -460,6 +467,7 @@ async fn concurrent_forwards_share_one_session_recovery() {
.expect("the resumed container is findable");
complete_resume(&resumed).await;
resumed.agent().wait_for_requests(4).await;
+ resumed.agent().completes_prompt().await;
resumed
};
@@ -673,16 +681,16 @@ async fn a_prompt_through_control_reaches_the_agent_without_announcing() {
let container = live_session(&service, &containers, id).await;
let announced_before = announcer.announced().len();
- service
- .control_event(
- id,
- ControlEvent {
- action: AgentAction::prompt("and now the docs"),
- actor: Some(sender()),
- },
- )
- .await
- .expect("prompting through control should succeed");
+ let prompted = service.control_event(
+ id,
+ ControlEvent {
+ action: AgentAction::prompt("and now the docs"),
+ actor: Some(sender()),
+ },
+ );
+ let agent = container.agent();
+ let (result, ()) = tokio::join!(prompted, agent.completes_prompt());
+ result.expect("prompting through control should succeed");
assert_eq!(
prompts(&container.agent()).len(),
@@ -702,16 +710,16 @@ async fn compact_through_control_reaches_opencode_as_a_slash_command() {
let id = AgentSessionId::new();
let container = live_session(&service, &containers, id).await;
- service
- .control_event(
- id,
- ControlEvent {
- action: AgentAction::Compact,
- actor: Some(sender()),
- },
- )
- .await
- .expect("compaction should reach the running agent");
+ let compacted = service.control_event(
+ id,
+ ControlEvent {
+ action: AgentAction::Compact,
+ actor: Some(sender()),
+ },
+ );
+ let agent = container.agent();
+ let (result, ()) = tokio::join!(compacted, agent.completes_prompt());
+ result.expect("compaction should reach the running agent");
assert_eq!(
prompts(&container.agent()),
@@ -784,6 +792,7 @@ async fn complete_bound_handshake(runtime: &ContainerMock) {
agent.completes_initialize(InitializeResponse::new(PROTOCOL_VERSION));
agent.wait_for_requests(2).await;
agent.opens_session(NewSessionResponse::new("acp-test"));
+ agent.completes_prompt().await;
}
/// As [`complete_bound_handshake`], for a session the runtime is restoring.
@@ -806,6 +815,7 @@ async fn complete_bound_resume(runtime: &ContainerMock) {
ClientRequest::ResumeSessionRequest(request) if request.session_id.to_string() == "acp-test"
));
agent.resumes_session(ResumeSessionResponse::new());
+ agent.completes_prompt().await;
}
/// Wait until a session has noticed that its transport ended.
@@ -918,9 +928,10 @@ async fn a_bound_session_stays_on_its_connection_until_it_drops() {
// Already bound: a second prompt goes straight down the same connection,
// with no second handshake to drive.
- prompt(&service, session.id, "and now the docs")
- .await
- .expect("a bound session needs no rebinding");
+ let prompted = prompt(&service, session.id, "and now the docs");
+ let agent = first.agent();
+ let (result, ()) = tokio::join!(prompted, agent.completes_prompt());
+ result.expect("a bound session needs no rebinding");
assert_eq!(
prompts(&first.agent()),
["fix the failing test", "and now the docs"]
@@ -1200,7 +1211,7 @@ async fn open_managed_session_spawns_at_the_users_default_size() {
let container = containers
.container(session_of(&containers))
.expect("the spawned container is findable");
- complete_handshake(&container).await;
+ complete_session_handshake(&container).await;
};
let (opened, _) = tokio::join!(open, drive);
let session = opened.expect("open should succeed");
diff --git a/crates/agent_harness/src/outbound/daytona/client.rs b/crates/agent_harness/src/outbound/daytona/client.rs
index 26e3fafa22..04b138a387 100644
--- a/crates/agent_harness/src/outbound/daytona/client.rs
+++ b/crates/agent_harness/src/outbound/daytona/client.rs
@@ -124,7 +124,7 @@ impl DaytonaClient {
/// Resource fields must be omitted: Daytona returns 400 ("Cannot specify
/// Sandbox resources when using a snapshot") if `cpu` / `memory` / `disk`
/// are sent with `snapshot`. Size is applied after start via [`Self::resize`].
- #[tracing::instrument(err, skip(self, env))]
+ #[tracing::instrument(name = "daytona.sandbox.create", err, skip(self, env))]
pub async fn create(&self, snapshot: &Snapshot, env: Env, labels: Labels) -> Result<String> {
let request = configuration_parameters(snapshot, env, labels);
let sandbox: SandboxDto = self
@@ -184,7 +184,7 @@ impl DaytonaClient {
}
/// Find one sandbox carrying the supplied label.
- #[tracing::instrument(err, skip(self))]
+ #[tracing::instrument(name = "daytona.sandbox.find", err, skip(self))]
pub async fn find_by_label(&self, label: &str, value: &str) -> Result<Option<String>> {
let labels = Labels::from(HashMap::from([(label.to_owned(), value.to_owned())]));
let filter = serde_json::to_string(&labels).map_err(DaytonaError::EncodeLabelFilter)?;
@@ -211,7 +211,7 @@ impl DaytonaClient {
}
/// Poll a sandbox until it has started or the deadline passes.
- #[tracing::instrument(err, skip(self))]
+ #[tracing::instrument(name = "daytona.sandbox.wait_started", err, skip(self))]
pub async fn wait_for_started(&self, sandbox_id: &str, timeout: Duration) -> Result<()> {
let deadline = Instant::now() + timeout;
loop {
@@ -373,7 +373,7 @@ impl DaytonaClient {
}
/// Execute one command in a sandbox.
- #[tracing::instrument(err, skip(self))]
+ #[tracing::instrument(name = "daytona.command.execute", err, skip(self, command))]
pub async fn exec(&self, sandbox_id: &str, command: &str, timeout: Duration) -> Result<String> {
let toolbox: ToolboxProxyUrlDto = self
.json(
@@ -451,7 +451,7 @@ impl DaytonaClient {
}
/// Poll the sidecar readiness endpoint until it succeeds.
- #[tracing::instrument(err, skip(self))]
+ #[tracing::instrument(name = "daytona.sidecar.wait_ready", err, skip(self))]
pub async fn wait_for_ping(
&self,
ping_url: &str,
@@ -461,6 +461,9 @@ impl DaytonaClient {
let deadline = Instant::now() + timeout;
loop {
let mut request = self.http.get(ping_url);
+ let mut trace_headers = reqwest::header::HeaderMap::new();
+ macro_tower_layers::inject_trace_headers(&mut trace_headers);
+ request = request.headers(trace_headers);
if let Some(token) = preview_token {
request = request.header("x-daytona-preview-token", token);
}
@@ -485,7 +488,10 @@ impl DaytonaClient {
request: reqwest::RequestBuilder,
operation: &'static str,
) -> Result<T> {
+ let mut trace_headers = reqwest::header::HeaderMap::new();
+ macro_tower_layers::inject_trace_headers(&mut trace_headers);
let response = request
+ .headers(trace_headers)
.bearer_auth(self.api_key.expose())
.send()
.await
diff --git a/crates/agent_harness/src/outbound/daytona/manager.rs b/crates/agent_harness/src/outbound/daytona/manager.rs
index 43c53355ce..9daf3f0c6b 100644
--- a/crates/agent_harness/src/outbound/daytona/manager.rs
+++ b/crates/agent_harness/src/outbound/daytona/manager.rs
@@ -8,6 +8,7 @@ use agent_session::domain::model::AgentSessionId;
use futures::{StreamExt as _, stream};
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_util::{sync::CancellationToken, task::TaskTracker};
+use tracing::Instrument as _;
use super::client::DaytonaClient;
use super::errors::DaytonaError;
@@ -110,6 +111,12 @@ impl DaytonaContainerManager {
}
}
+ #[tracing::instrument(
+ name = "agent.container.boot",
+ err,
+ skip(self, id),
+ fields(agent.container.provider = "daytona", agent.container.id = id.as_str())
+ )]
async fn bring_up(&self, id: &DaytonaSandboxId) -> Result<DaytonaContainer> {
self.client
.wait_for_started(id.as_str(), provision::ENSURE_TIMEOUT)
@@ -122,6 +129,7 @@ impl DaytonaContainerManager {
&provision::ensure_ready_command(),
provision::ENSURE_TIMEOUT,
)
+ .instrument(tracing::info_span!("agent.container.ensure_ready"))
.await
.map_err(unavailable)?;
tracing::info!(sandbox_id = %id.as_str(), %output, "readiness recipe finished");
@@ -139,7 +147,10 @@ impl DaytonaContainerManager {
)
.await
.map_err(unavailable)?;
- let socket = dial_sidecar(&preview).await.map_err(unavailable)?;
+ let socket = dial_sidecar(&preview)
+ .instrument(tracing::info_span!("agent.container.websocket_connect"))
+ .await
+ .map_err(unavailable)?;
if !self.managed.containers.activate(id, Instant::now()) {
return Err(HarnessError::Container(
"sandbox is no longer managed".to_owned(),
@@ -336,7 +347,12 @@ impl DaytonaContainerManager {
impl ContainerManager for DaytonaContainerManager {
type Transport = DaytonaContainer;
- #[tracing::instrument(err, skip(self))]
+ #[tracing::instrument(
+ name = "agent.container.spawn",
+ err,
+ skip(self),
+ fields(agent.container.provider = "daytona")
+ )]
async fn spawn(&self, command: SpawnContainer) -> Result<DaytonaContainer> {
let SpawnContainer {
session_id,
diff --git a/crates/agent_harness/src/outbound/namespace/client.rs b/crates/agent_harness/src/outbound/namespace/client.rs
index 62b1477370..72c9ee39e2 100644
--- a/crates/agent_harness/src/outbound/namespace/client.rs
+++ b/crates/agent_harness/src/outbound/namespace/client.rs
@@ -316,9 +316,12 @@ impl NamespaceClient {
B: Serialize + ?Sized,
{
let url = format!("{}/{service}/{method}", host.trim_end_matches('/'));
+ let mut trace_headers = reqwest::header::HeaderMap::new();
+ macro_tower_layers::inject_trace_headers(&mut trace_headers);
let response = self
.http
.post(url)
+ .headers(trace_headers)
.bearer_auth(self.token.expose())
.json(body)
.send()
diff --git a/crates/agent_harness/src/outbound/namespace/manager.rs b/crates/agent_harness/src/outbound/namespace/manager.rs
index f6f72ad7e3..249fbfc113 100644
--- a/crates/agent_harness/src/outbound/namespace/manager.rs
+++ b/crates/agent_harness/src/outbound/namespace/manager.rs
@@ -3,6 +3,7 @@ use std::sync::Arc;
use agent_runtime_protocol::domain::ports::Transport;
use agent_runtime_protocol::domain::schema::v0::{ToRuntimeMessage, ToServerMessage};
use agent_session::domain::model::{AgentSessionId, SandboxSize};
+use tracing::Instrument as _;
use super::client::NamespaceClient;
use super::errors::NamespaceError;
@@ -42,6 +43,12 @@ impl NamespaceContainerManager {
}
}
+ #[tracing::instrument(
+ name = "agent.container.boot",
+ err,
+ skip(self),
+ fields(agent.container.provider = "namespace", agent.container.id = %instance.id)
+ )]
async fn bring_up(&self, instance: Instance) -> Result<NamespaceContainer> {
self.client
.wait_until_ready(&instance.id, provision::ENSURE_TIMEOUT)
@@ -53,6 +60,7 @@ impl NamespaceContainerManager {
&instance,
&["bash", "-lc", &provision::ensure_ready_command()],
)
+ .instrument(tracing::info_span!("agent.container.ensure_ready"))
.await
.map_err(unavailable)?;
if output.exit_code != 0 {
@@ -69,7 +77,10 @@ impl NamespaceContainerManager {
.create_ingress(&instance.id, provision::SIDECAR_PORT)
.await
.map_err(unavailable)?;
- let socket = dial_sidecar(&sidecar_url).await.map_err(unavailable)?;
+ let socket = dial_sidecar(&sidecar_url)
+ .instrument(tracing::info_span!("agent.container.websocket_connect"))
+ .await
+ .map_err(unavailable)?;
Ok(NamespaceContainer {
instance: Arc::new(instance),
@@ -82,7 +93,12 @@ impl NamespaceContainerManager {
impl ContainerManager for NamespaceContainerManager {
type Transport = NamespaceContainer;
- #[tracing::instrument(err, skip(self))]
+ #[tracing::instrument(
+ name = "agent.container.spawn",
+ err,
+ skip(self),
+ fields(agent.container.provider = "namespace")
+ )]
async fn spawn(&self, command: SpawnContainer) -> Result<Self::Transport> {
let container = ContainerSpec {
image_ref: self.image_ref.clone(),
diff --git a/crates/agent_harness/src/outbound/sidecar.rs b/crates/agent_harness/src/outbound/sidecar.rs
index c450c0c4a3..bdaea15f93 100644
--- a/crates/agent_harness/src/outbound/sidecar.rs
+++ b/crates/agent_harness/src/outbound/sidecar.rs
@@ -17,6 +17,8 @@ use tokio::io::{AsyncRead, AsyncWrite};
use tokio::sync::{mpsc, oneshot};
use tokio_tungstenite::WebSocketStream;
use tokio_tungstenite::tungstenite::Message;
+use tracing::Instrument as _;
+use tracing::instrument::WithSubscriber as _;
#[cfg(test)]
mod test;
@@ -26,17 +28,18 @@ mod test;
/// Not `Clone` itself - `recv` has to be exclusive - so containers share one
/// through an `Arc`, which is what makes them cloneable.
pub struct SidecarTransport {
- outbound: mpsc::UnboundedSender<Outbound>,
+ outbound: mpsc::UnboundedSender<OutboundFrame>,
inbound: mpsc::UnboundedReceiver<ToServerMessage>,
}
/// The sidecar's sending half.
pub struct SidecarSender {
- outbound: mpsc::UnboundedSender<Outbound>,
+ outbound: mpsc::UnboundedSender<OutboundFrame>,
}
-struct Outbound {
+struct OutboundFrame {
frame: RawJsonRpcMessage,
+ parent: tracing::Span,
completed: oneshot::Sender<Result<(), TransportError>>,
}
@@ -72,7 +75,7 @@ impl SidecarTransport {
};
let _ = inbound_tx.send(ready);
- tokio::spawn(pump(socket, outbound_rx, inbound_tx, on_frame));
+ tokio::spawn(pump(socket, outbound_rx, inbound_tx, on_frame).with_current_subscriber());
Self {
outbound: outbound_tx,
@@ -105,9 +108,14 @@ impl TransportSender<ToRuntimeMessage> for SidecarSender {
return Ok(());
};
+ let parent = tracing::Span::current();
let (completed, result) = oneshot::channel();
self.outbound
- .send(Outbound { frame, completed })
+ .send(OutboundFrame {
+ frame,
+ parent,
+ completed,
+ })
.map_err(|_| TransportError::Client("the sidecar connection is closed".to_owned()))?;
result
.await
@@ -118,7 +126,7 @@ impl TransportSender<ToRuntimeMessage> for SidecarSender {
/// Relay frames until either side closes.
async fn pump<Socket, Observer>(
socket: WebSocketStream<Socket>,
- mut outbound: mpsc::UnboundedReceiver<Outbound>,
+ mut outbound: mpsc::UnboundedReceiver<OutboundFrame>,
inbound: mpsc::UnboundedSender<ToServerMessage>,
on_frame: Observer,
) where
@@ -130,7 +138,10 @@ async fn pump<Socket, Observer>(
loop {
tokio::select! {
outgoing = outbound.recv() => {
- let Some(Outbound { frame, completed }) = outgoing else { break };
+ let Some(OutboundFrame { frame, parent, completed }) = outgoing else { break };
+ let method = frame_method(&frame);
+ // Serialization failure is fatal for this session because the
+ // caller's action cannot reach the agent.
let json = match serde_json::to_string(&frame) {
Ok(json) => json,
Err(error) => {
@@ -141,10 +152,32 @@ async fn pump<Socket, Observer>(
continue;
}
};
- if let Err(error) = socket_tx.send(Message::Text(json.into())).await {
- let _ = completed.send(Err(TransportError::Client(error.to_string())));
+ let send = socket_tx.send(Message::Text(json.into()));
+ let span = tracing::info_span!(
+ parent: &parent,
+ "agent.acp.websocket_send",
+ network.protocol.name = "websocket",
+ rpc.system.name = tracing::field::Empty,
+ rpc.method = tracing::field::Empty,
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ if let Some(method) = method {
+ span.record("rpc.system.name", "jsonrpc");
+ span.record("rpc.method", method);
+ }
+ let result = send
+ .instrument(span.clone())
+ .await
+ .map_err(|error| format!("sending an ACP websocket frame failed: {error}"));
+ if let Err(error) = result {
+ span.record("otel.status_code", "ERROR");
+ span.record("otel.status_description", tracing::field::display(&error));
+ drop(span);
+ let _ = completed.send(Err(TransportError::Client(error)));
break;
}
+ drop(span);
let _ = completed.send(Ok(()));
}
incoming = socket_rx.next() => {
@@ -181,3 +214,11 @@ async fn pump<Socket, Observer>(
tracing::debug!("sidecar pump finished");
}
+
+fn frame_method(frame: &RawJsonRpcMessage) -> Option<&str> {
+ match frame {
+ RawJsonRpcMessage::Request(request) => Some(request.method.as_ref()),
+ RawJsonRpcMessage::Notification(notification) => Some(notification.method.as_ref()),
+ RawJsonRpcMessage::Response(_) => None,
+ }
+}
diff --git a/crates/agent_harness/src/outbound/sidecar/test.rs b/crates/agent_harness/src/outbound/sidecar/test.rs
index 519ea9190d..0e8c8d7b24 100644
--- a/crates/agent_harness/src/outbound/sidecar/test.rs
+++ b/crates/agent_harness/src/outbound/sidecar/test.rs
@@ -2,9 +2,9 @@ use agent_client_protocol::schema::v1::{PromptRequest, RequestId, SessionId};
use agent_client_protocol::{JsonRpcMessage, RawJsonRpcMessage};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
-use tokio::net::{TcpListener, TcpStream};
use super::*;
+use tokio::net::{TcpListener, TcpStream};
/// One ACP frame built from the protocol's typed request.
fn frame() -> RawJsonRpcMessage {
diff --git a/crates/agent_harness/src/testing/helpers/agent.rs b/crates/agent_harness/src/testing/helpers/agent.rs
index ce14175e93..3fed915fd3 100644
--- a/crates/agent_harness/src/testing/helpers/agent.rs
+++ b/crates/agent_harness/src/testing/helpers/agent.rs
@@ -1,10 +1,11 @@
//! A fake agent process, speaking only raw ACP.
+use std::collections::VecDeque;
use std::sync::{Arc, Mutex};
use agent_client_protocol::schema::v1::{
- ClientNotification, ClientRequest, InitializeResponse, NewSessionResponse, RequestId,
- ResumeSessionResponse,
+ ClientNotification, ClientRequest, InitializeResponse, NewSessionResponse, PromptResponse,
+ RequestId, ResumeSessionResponse, StopReason,
};
use agent_client_protocol::{
Error as AcpError, JsonRpcMessage, JsonRpcNotification, JsonRpcResponse, RawJsonRpcMessage,
@@ -32,6 +33,7 @@ struct Progress {
/// Requests received and not yet answered.
initializing: Option<RequestId>,
opening: Option<RequestId>,
+ prompts: VecDeque<RequestId>,
}
/// Parse a frame into whichever ACP message enum the caller asked for.
@@ -119,6 +121,21 @@ impl FakeAgent {
self.sends_reply(id, response);
}
+ /// Wait for and complete the next `session/prompt` request.
+ pub async fn completes_prompt(&self) {
+ let mut received = self.received.subscribe();
+ let id = loop {
+ if let Some(id) = self.lock_progress().prompts.pop_front() {
+ break id;
+ }
+ received
+ .changed()
+ .await
+ .expect("fake agent frame history should remain open");
+ };
+ self.sends_reply(id, PromptResponse::new(StopReason::EndTurn));
+ }
+
/// Refuse the `session/new` this agent received.
///
/// # Panics
@@ -248,6 +265,15 @@ impl FakeAgent {
);
progress.opening = Some(request.id.clone());
}
+ ClientRequest::PromptRequest(_) => {
+ assert_eq!(
+ progress.stage,
+ Stage::SessionOpen,
+ "harness sent {} before its ACP session existed",
+ request.method
+ );
+ progress.prompts.push_back(request.id.clone());
+ }
other => assert_eq!(
progress.stage,
Stage::SessionOpen,
diff --git a/crates/agent_runtime_protocol/Cargo.toml b/crates/agent_runtime_protocol/Cargo.toml
index ca8554a6f0..546b170609 100644
--- a/crates/agent_runtime_protocol/Cargo.toml
+++ b/crates/agent_runtime_protocol/Cargo.toml
@@ -24,6 +24,7 @@ serde_json = { workspace = true }
specta = { workspace = true }
specta-serde = { workspace = true }
specta-typescript = { workspace = true }
+strum = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt", "sync"], optional = true }
tokio-tungstenite = { workspace = true, optional = true }
diff --git a/crates/agent_runtime_protocol/src/domain/action.rs b/crates/agent_runtime_protocol/src/domain/action.rs
index 7a218cca42..e6e6241d41 100644
--- a/crates/agent_runtime_protocol/src/domain/action.rs
+++ b/crates/agent_runtime_protocol/src/domain/action.rs
@@ -127,9 +127,10 @@ impl AgentSetModelAction {
}
/// One thing a caller wants an agent to do.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, strum::AsRefStr)]
#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
#[serde(tag = "type", rename_all = "camelCase")]
+#[strum(serialize_all = "snake_case")]
pub enum AgentAction {
/// Send the agent a prompt.
Prompt(AgentPromptAction),
diff --git a/crates/agent_session/Cargo.toml b/crates/agent_session/Cargo.toml
index cc8b485cf1..71a8c17972 100644
--- a/crates/agent_session/Cargo.toml
+++ b/crates/agent_session/Cargo.toml
@@ -43,6 +43,7 @@ serde = { workspace = true }
serde_json = { workspace = true }
strum = { workspace = true }
tokio = { workspace = true, features = ["rt", "sync"] }
+tokio-util = { workspace = true, features = ["rt"] }
tracing = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/agent_session/src/domain/error.rs b/crates/agent_session/src/domain/error.rs
index 7778d0c352..b233655576 100644
--- a/crates/agent_session/src/domain/error.rs
+++ b/crates/agent_session/src/domain/error.rs
@@ -22,6 +22,10 @@ pub enum AgentSessionError {
"agent session {0} cannot be restored because the agent supports neither session/resume nor session/load"
)]
ResumeUnsupported(AgentSessionId),
+ #[error("agent session {0} action delivery timed out")]
+ DeliveryTimedOut(AgentSessionId),
+ #[error("agent session {0} log persistence timed out")]
+ LogTimedOut(AgentSessionId),
#[error(transparent)]
Transport(#[from] TransportError),
#[error(transparent)]
diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rs
index 5f1d845875..61c2abb465 100644
--- a/crates/agent_session/src/domain/service.rs
+++ b/crates/agent_session/src/domain/service.rs
@@ -37,7 +37,9 @@ use dashmap::mapref::entry::Entry;
use entity_access::domain::models::{EntityAccessReceipt, EntityType, OwnerAccessLevel};
use macro_user_id::user_id::MacroUserIdStr;
use macro_uuid::Uuid;
-use tokio::sync::{mpsc, oneshot};
+use tokio::sync::{Mutex, mpsc, oneshot, watch};
+use tokio_util::{sync::CancellationToken, task::TaskTracker};
+use tracing::instrument::WithSubscriber as _;
use bots::domain::models::BotId;
@@ -53,11 +55,54 @@ use super::ports::{
AgentSessionRealtime, AgentSessionRepo, NoOpAgentSessionNameGenerator,
};
use super::session::actors::{SessionActor, SessionCommand, Stepped};
+use super::session::{CloseReason, Input};
/// Buffered not-yet-accepted commands per session actor.
const COMMAND_BUFFER: usize = 1028;
+/// Persistence may delay lifecycle teardown, but never indefinitely.
+const SESSION_PERSIST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
+
+struct ActiveSession {
+ commands: Option<mpsc::Sender<SessionCommand>>,
+ stopped: watch::Receiver<bool>,
+ marker: Arc<()>,
+ deleting: bool,
+ stopping: bool,
+}
+
+type ActiveSessions = DashMap<AgentSessionId, ActiveSession>;
-type ActiveSessions = DashMap<AgentSessionId, mpsc::Sender<SessionCommand>>;
+struct AttachReservation {
+ active: Arc<ActiveSessions>,
+ id: AgentSessionId,
+ marker: Arc<()>,
+ stopped: Option<watch::Sender<bool>>,
+ committed: bool,
+}
+
+impl AttachReservation {
+ fn commit(mut self) -> (Arc<()>, watch::Sender<bool>) {
+ self.committed = true;
+ (
+ self.marker.clone(),
+ self.stopped.take().expect("reservation owns stop signal"),
+ )
+ }
+}
+
+impl Drop for AttachReservation {
+ fn drop(&mut self) {
+ if self.committed {
+ return;
+ }
+ if let Some(stopped) = self.stopped.take() {
+ let _ = stopped.send(true);
+ }
+ self.active.remove_if(&self.id, |_, active| {
+ Arc::ptr_eq(&active.marker, &self.marker) && !active.stopping
+ });
+ }
+}
/// Durable and live use cases for agent sessions.
#[cfg_attr(feature = "test-utils", mockall::automock)]
@@ -167,12 +212,16 @@ pub trait AgentSessionService: Send + Sync + 'static {
/// `Folds` answers "what messages does this session's log derive" -
/// `agent_fold` folding the log on read - and `Rt` streams each frame to
/// whoever is watching the session's channel right now.
+#[derive(Clone)]
pub struct AgentSessionServiceImpl<R, Folds, Rt, Namer = NoOpAgentSessionNameGenerator> {
repo: R,
folds: Folds,
realtime: Rt,
name_generator: Namer,
active: Arc<ActiveSessions>,
+ tasks: TaskTracker,
+ cancellation: CancellationToken,
+ lifecycle: Arc<Mutex<()>>,
}
impl<R, Folds, Rt> AgentSessionServiceImpl<R, Folds, Rt> {
@@ -189,6 +238,9 @@ impl<R, Folds, Rt> AgentSessionServiceImpl<R, Folds, Rt> {
realtime,
name_generator: NoOpAgentSessionNameGenerator,
active: Arc::new(DashMap::new()),
+ tasks: TaskTracker::new(),
+ cancellation: CancellationToken::new(),
+ lifecycle: Arc::new(Mutex::new(())),
}
}
}
@@ -206,29 +258,80 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
realtime: self.realtime,
name_generator,
active: self.active,
+ tasks: self.tasks,
+ cancellation: self.cancellation,
+ lifecycle: self.lifecycle,
}
}
- fn register_transport<Connector>(
+ /// Stop active actors and wait for their tasks to release their transports.
+ pub async fn shutdown(&self) {
+ let lifecycle = self.lifecycle.lock().await;
+ self.cancellation.cancel();
+ for mut session in self.active.iter_mut() {
+ session.commands.take();
+ session.stopping = true;
+ }
+ self.tasks.close();
+ drop(lifecycle);
+ self.tasks.wait().await;
+ self.active.clear();
+ }
+
+ async fn reserve_attach(&self, id: AgentSessionId) -> Result<AttachReservation> {
+ let _lifecycle = self.lifecycle.lock().await;
+ if self.cancellation.is_cancelled() {
+ return Err(AgentSessionError::Disconnected(id));
+ }
+ let (stopped_tx, stopped) = watch::channel(false);
+ let marker = Arc::new(());
+ match self.active.entry(id) {
+ Entry::Occupied(_) => Err(AgentSessionError::AlreadyConnected(id)),
+ Entry::Vacant(entry) => {
+ entry.insert(ActiveSession {
+ commands: None,
+ stopped,
+ marker: marker.clone(),
+ deleting: false,
+ stopping: false,
+ });
+ Ok(AttachReservation {
+ active: self.active.clone(),
+ id,
+ marker,
+ stopped: Some(stopped_tx),
+ committed: false,
+ })
+ }
+ }
+ }
+
+ async fn activate_reserved<Connector>(
&self,
- id: AgentSessionId,
- acp_session_id: Option<SessionId>,
- workspace: String,
+ session: AgentSession,
attachment: RuntimeAttachment<Connector>,
+ reservation: AttachReservation,
) -> Result<()>
where
R: AgentSessionRepo + AgentSessionLogRepo + Clone,
Rt: AgentSessionRealtime + Clone + Send + Sync + 'static,
Connector: AgentConnector,
{
+ let _lifecycle = self.lifecycle.lock().await;
+ let id = session.id;
+ if self.cancellation.is_cancelled() {
+ return Err(AgentSessionError::Disconnected(id));
+ }
let (commands, command_rx) = mpsc::channel(COMMAND_BUFFER);
-
- match self.active.entry(id) {
- Entry::Occupied(_) => return Err(AgentSessionError::AlreadyConnected(id)),
- Entry::Vacant(entry) => {
- entry.insert(commands.clone());
- }
+ let Some(mut active) = self.active.get_mut(&id) else {
+ return Err(AgentSessionError::Disconnected(id));
+ };
+ if !Arc::ptr_eq(&active.marker, &reservation.marker) || active.stopping {
+ return Err(AgentSessionError::Disconnected(id));
}
+ active.commands = Some(commands.clone());
+ drop(active);
+ let (marker, stopped_tx) = reservation.commit();
// The actor owns this session's log writes, so it gets the live writer
// rather than the bare repository - see module docs. Its fold starts
@@ -238,14 +341,23 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
let logs = LiveSessionLogWriter::new(self.repo.clone(), self.realtime.clone());
let actor = SessionActor::new(
id,
- acp_session_id,
- workspace,
+ session.acp_session_id,
+ session.workspace,
attachment.connector,
logs,
command_rx,
attachment.handshake,
);
- tokio::spawn(run_session(actor, Arc::downgrade(&self.active), commands));
+ self.tasks.spawn(
+ run_session(
+ actor,
+ Arc::downgrade(&self.active),
+ marker,
+ stopped_tx,
+ self.cancellation.clone(),
+ )
+ .with_current_subscriber(),
+ );
Ok(())
}
@@ -259,31 +371,70 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
let commands = self
.active
.get(&id)
- .map(|entry| entry.value().clone())
+ .and_then(|entry| entry.commands.clone())
.ok_or(AgentSessionError::Disconnected(id))?;
let (completed, result) = oneshot::channel();
+ let span = tracing::info_span!(
+ "agent.session.command",
+ agent.session.id = %id,
+ agent.action.name = action.as_ref(),
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
if commands
.send(SessionCommand {
user_id,
action,
action_id,
completed,
+ span,
})
.await
.is_err()
{
- self.active
- .remove_if(&id, |_, active| active.same_channel(&commands));
return Err(AgentSessionError::Disconnected(id));
}
match result.await {
Ok(result) => result,
- Err(_) => {
- self.active
- .remove_if(&id, |_, active| active.same_channel(&commands));
- Err(AgentSessionError::Disconnected(id))
+ Err(_) => Err(AgentSessionError::Disconnected(id)),
+ }
+ }
+
+ fn begin_stop(&self, id: AgentSessionId, deleting: bool) -> (watch::Receiver<bool>, Arc<()>) {
+ match self.active.entry(id) {
+ Entry::Occupied(mut entry) => {
+ let active = entry.get_mut();
+ let attaching = active.commands.is_none() && !active.stopping;
+ active.commands.take();
+ active.deleting |= deleting;
+ active.stopping = true;
+ let stopped = if attaching {
+ let (_stopped_tx, stopped) = watch::channel(true);
+ stopped
+ } else {
+ active.stopped.clone()
+ };
+ (stopped, active.marker.clone())
}
+ Entry::Vacant(entry) => {
+ let (_stopped_tx, stopped) = watch::channel(true);
+ let marker = Arc::new(());
+ entry.insert(ActiveSession {
+ commands: None,
+ stopped: stopped.clone(),
+ marker: marker.clone(),
+ deleting,
+ stopping: true,
+ });
+ (stopped, marker)
+ }
+ }
+ }
+
+ async fn wait_stopped(mut stopped: watch::Receiver<bool>) {
+ if !*stopped.borrow() {
+ let _ = stopped.wait_for(|value| *value).await;
}
}
}
@@ -341,27 +492,40 @@ where
}
async fn delete_session(&self, id: AgentSessionId) -> Result<()> {
- self.repo.delete(id).await
+ let (stopped, marker) = self.begin_stop(id, true);
+ Self::wait_stopped(stopped).await;
+ let result = self.repo.delete(id).await;
+ self.active
+ .remove_if(&id, |_, active| Arc::ptr_eq(&active.marker, &marker));
+ result
}
async fn close_session(&self, id: AgentSessionId) -> Result<()> {
// Dropping the sender is the whole operation: the actor's next step
// reads `None` from its command channel, treats it as `Abandoned`, and
// tears the transport down on its way out.
- self.active.remove(&id);
+ let (stopped, marker) = self.begin_stop(id, false);
+ Self::wait_stopped(stopped).await;
+ self.active.remove_if(&id, |_, active| {
+ Arc::ptr_eq(&active.marker, &marker) && !active.deleting
+ });
Ok(())
}
async fn mark_disconnected(&self, id: AgentSessionId) -> Result<()> {
let mut logs = LiveSessionLogWriter::new(self.repo.clone(), self.realtime.clone());
- logs.append(AgentSessionLog {
- agent_session_id: id,
- user_id: None,
- content: Message::ToServer(ToServerMessage::Event {
- event: SystemEvent::Disconnected,
+ tokio::time::timeout(
+ SESSION_PERSIST_TIMEOUT,
+ logs.append(AgentSessionLog {
+ agent_session_id: id,
+ user_id: None,
+ content: Message::ToServer(ToServerMessage::Event {
+ event: SystemEvent::Disconnected,
+ }),
}),
- })
+ )
.await
+ .unwrap_or(Err(AgentSessionError::LogTimedOut(id)))
}
async fn attach_session<Connector>(
@@ -372,13 +536,10 @@ where
where
Connector: AgentConnector,
{
+ let reservation = self.reserve_attach(id).await?;
let session = self.repo.get(id).await?;
- self.register_transport(
- session.id,
- session.acp_session_id,
- session.workspace,
- attachment,
- )
+ self.activate_reserved(session, attachment, reservation)
+ .await
}
async fn send_action(
@@ -758,12 +919,24 @@ where
async fn run_session<Connector, Logs>(
mut actor: SessionActor<Connector, Logs>,
active: std::sync::Weak<ActiveSessions>,
- commands: mpsc::Sender<SessionCommand>,
+ marker: Arc<()>,
+ stopped: watch::Sender<bool>,
+ cancellation: CancellationToken,
) where
Connector: AgentConnector,
Logs: AgentSessionLogWriter + AgentSessionRepo,
{
- while actor.step().await == Stepped::Continue {}
+ loop {
+ let input = tokio::select! {
+ biased;
+ () = cancellation.cancelled() => Input::Closed(CloseReason::Abandoned),
+ input = actor.next_input() => input,
+ };
+ let stepped = actor.dispatch(input).await;
+ if stepped == Stepped::Stopped {
+ break;
+ }
+ }
// Refuse late commands before releasing the registry entry, so a caller
// cannot enqueue into an actor that will never step again.
@@ -772,7 +945,10 @@ async fn run_session<Connector, Logs>(
// Tear down the old transport before allowing another actor to attach.
drop(actor);
+ let _ = stopped.send(true);
if let Some(active) = active.upgrade() {
- active.remove_if(&id, |_, current| current.same_channel(&commands));
+ active.remove_if(&id, |_, current| {
+ current.commands.is_some() && Arc::ptr_eq(&current.marker, &marker)
+ });
}
}
diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rs
index 5ec04927a2..e98d85cb48 100644
--- a/crates/agent_session/src/domain/service/test.rs
+++ b/crates/agent_session/src/domain/service/test.rs
@@ -1,14 +1,22 @@
use super::*;
-use crate::domain::model::{DEFAULT_AGENT_SESSION_NAME, Message};
+use crate::PROTOCOL_VERSION;
+use crate::domain::model::{DEFAULT_AGENT_SESSION_NAME, Message, SessionBot};
use crate::domain::ports::NoOpRealtime;
+use crate::domain::session::HandshakeStatus;
use crate::testing::{InMemoryAgentSessionRepo, RecordingRealtime, test_agent_session};
use agent_fold::domain::fold::fold;
use agent_fold::domain::service::FoldedMessageService;
use agent_fold::testing::{TURN, parse_log_as, test_session};
-use agent_runtime_protocol::domain::schema::v0::ToServerMessage;
+use agent_runtime_protocol::domain::ports::{
+ Transport, TransportError, TransportReceiver, TransportSender,
+};
+use agent_runtime_protocol::domain::schema::v0::ToRuntimeMessage;
+use agent_runtime_protocol::domain::schema::v0::{AcpMessage, ToServerMessage};
use entity_access::domain::models::{EntityAccessReceipt, EntityType, OwnerAccessLevel};
use macro_uuid::Uuid;
use std::sync::{Arc, Mutex};
+use tokio::sync::Notify;
+use tracing::instrument::WithSubscriber as _;
struct Fixture {
service: AgentSessionServiceImpl<
@@ -92,6 +100,64 @@ impl AgentSessionRealtime for RenameRealtime {
}
}
+struct PendingTransport;
+
+#[derive(Clone)]
+struct PendingSender;
+
+struct PendingReceiver;
+
+struct RecordingTransport {
+ outbound: mpsc::Sender<ToRuntimeMessage>,
+ inbound: mpsc::Receiver<ToServerMessage>,
+}
+
+#[derive(Clone)]
+struct RecordingSender(mpsc::Sender<ToRuntimeMessage>);
+
+impl Transport<ToRuntimeMessage, ToServerMessage> for RecordingTransport {
+ type Sender = RecordingSender;
+ type Receiver = mpsc::Receiver<ToServerMessage>;
+
+ fn split(self) -> (Self::Sender, Self::Receiver) {
+ (RecordingSender(self.outbound), self.inbound)
+ }
+}
+
+impl TransportSender<ToRuntimeMessage> for RecordingSender {
+ async fn send(&self, message: ToRuntimeMessage) -> std::result::Result<(), TransportError> {
+ self.0
+ .send(message)
+ .await
+ .map_err(|_| TransportError::Client("test receiver closed".to_owned()))
+ }
+}
+
+#[derive(Clone)]
+struct BlockingPromptLogs {
+ repo: InMemoryAgentSessionRepo,
+ entered: Arc<Notify>,
+ release: Arc<Notify>,
+ hang_disconnect: bool,
+}
+
+impl AgentSessionLogWriter for BlockingPromptLogs {
+ async fn append(&mut self, log: AgentSessionLog) -> Result<()> {
+ let is_prompt = matches!(
+ &log.content,
+ Message::ToRuntime(ToRuntimeMessage::Acp(AcpMessage(
+ agent_client_protocol::RawJsonRpcMessage::Request(request)
+ ))) if request.method.as_ref() == "session/prompt"
+ );
+ if is_prompt {
+ self.entered.notify_one();
+ self.release.notified().await;
+ }
+ AgentSessionLogRepo::create(&self.repo, log).await?;
+ Ok(())
+ }
+}
+
fn owner_access(session: AgentSessionId) -> EntityAccessReceipt<OwnerAccessLevel> {
EntityAccessReceipt::dangerously_assert_internal_user(
&session.as_uuid().to_string(),
@@ -257,6 +323,420 @@ async fn background_naming_does_not_overwrite_a_manual_name() {
);
}
+impl AgentSessionRepo for BlockingPromptLogs {
+ async fn create(&self, params: CreateAgentSessionParams) -> Result<AgentSession> {
+ AgentSessionRepo::create(&self.repo, params).await
+ }
+
+ async fn get(&self, id: AgentSessionId) -> Result<AgentSession> {
+ self.repo.get(id).await
+ }
+
+ async fn session_bot(&self, id: BotId) -> Result<SessionBot> {
+ self.repo.session_bot(id).await
+ }
+
+ async fn find_for_channel(
+ &self,
+ thread_id: Option<Uuid>,
+ bot_id: Option<BotId>,
+ ) -> Result<ChannelSession> {
+ self.repo.find_for_channel(thread_id, bot_id).await
+ }
+
+ async fn set_acp_session_id(
+ &self,
+ id: AgentSessionId,
+ acp_session_id: SessionId,
+ ) -> Result<()> {
+ self.repo.set_acp_session_id(id, acp_session_id).await
+ }
+
+ async fn set_model(&self, id: AgentSessionId, model: &str) -> Result<()> {
+ self.repo.set_model(id, model).await
+ }
+
+ async fn set_name(&self, id: AgentSessionId, name: &str) -> Result<()> {
+ self.repo.set_name(id, name).await
+ }
+
+ async fn set_name_if_default(&self, id: AgentSessionId, name: &str) -> Result<bool> {
+ self.repo.set_name_if_default(id, name).await
+ }
+
+ async fn set_sandbox_size(&self, id: AgentSessionId, size: SandboxSize) -> Result<()> {
+ self.repo.set_sandbox_size(id, size).await
+ }
+
+ async fn user_sandbox_size(&self, user_id: &MacroUserIdStr<'static>) -> Result<SandboxSize> {
+ self.repo.user_sandbox_size(user_id).await
+ }
+
+ async fn set_user_sandbox_size(
+ &self,
+ user_id: &MacroUserIdStr<'static>,
+ size: SandboxSize,
+ ) -> Result<()> {
+ self.repo.set_user_sandbox_size(user_id, size).await
+ }
+
+ async fn delete(&self, id: AgentSessionId) -> Result<()> {
+ self.repo.delete(id).await
+ }
+}
+
+impl AgentSessionLogRepo for BlockingPromptLogs {
+ async fn create(&self, log: AgentSessionLog) -> Result<StoredAgentSessionLog> {
+ if self.hang_disconnect
+ && matches!(
+ &log.content,
+ Message::ToServer(ToServerMessage::Event {
+ event: SystemEvent::Disconnected
+ })
+ )
+ {
+ return std::future::pending().await;
+ }
+ AgentSessionLogRepo::create(&self.repo, log).await
+ }
+
+ async fn list_by_session(
+ &self,
+ agent_session_id: AgentSessionId,
+ ) -> Result<Vec<StoredAgentSessionLog>> {
+ AgentSessionLogRepo::list_by_session(&self.repo, agent_session_id).await
+ }
+}
+
+impl Transport<ToRuntimeMessage, ToServerMessage> for PendingTransport {
+ type Sender = PendingSender;
+ type Receiver = PendingReceiver;
+
+ fn split(self) -> (Self::Sender, Self::Receiver) {
+ (PendingSender, PendingReceiver)
+ }
+}
+
+impl TransportSender<ToRuntimeMessage> for PendingSender {
+ async fn send(&self, _message: ToRuntimeMessage) -> std::result::Result<(), TransportError> {
+ Ok(())
+ }
+}
+
+impl TransportReceiver<ToServerMessage> for PendingReceiver {
+ async fn recv(&mut self) -> std::result::Result<Option<ToServerMessage>, TransportError> {
+ std::future::pending().await
+ }
+}
+
+async fn open_test_session(
+ inbound: &mpsc::Sender<ToServerMessage>,
+ outbound: &mut mpsc::Receiver<ToRuntimeMessage>,
+ session: AgentSessionId,
+) {
+ inbound
+ .send(ToServerMessage::Event {
+ event: SystemEvent::AcpReady,
+ })
+ .await
+ .unwrap();
+ let _initialize = outbound.recv().await.expect("initialize request");
+ inbound
+ .send(ToServerMessage::Acp(AcpMessage(
+ agent_client_protocol::RawJsonRpcMessage::response(
+ agent_client_protocol::schema::v1::RequestId::Str(format!(
+ "agent_session:{session}:0"
+ )),
+ Ok(serde_json::to_value(
+ agent_client_protocol::schema::v1::InitializeResponse::new(PROTOCOL_VERSION),
+ )
+ .unwrap()),
+ ),
+ )))
+ .await
+ .unwrap();
+ let _open = outbound.recv().await.expect("session/new request");
+ inbound
+ .send(ToServerMessage::Acp(AcpMessage(
+ agent_client_protocol::RawJsonRpcMessage::response(
+ agent_client_protocol::schema::v1::RequestId::Str(format!(
+ "agent_session:{session}:1"
+ )),
+ Ok(serde_json::to_value(
+ agent_client_protocol::schema::v1::NewSessionResponse::new("acp-1"),
+ )
+ .unwrap()),
+ ),
+ )))
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn a_stopping_lifecycle_entry_blocks_a_second_attach() {
+ let fx = fixture();
+ let (_stopped, marker) = fx.service.begin_stop(fx.session, false);
+
+ let result = fx
+ .service
+ .attach_session(fx.session, RuntimeAttachment::solo(PendingTransport))
+ .await;
+
+ assert!(matches!(result, Err(AgentSessionError::AlreadyConnected(id)) if id == fx.session));
+ fx.service.active.remove_if(&fx.session, |_, active| {
+ Arc::ptr_eq(&active.marker, &marker)
+ });
+}
+
+#[tokio::test]
+async fn close_claiming_an_attach_reservation_prevents_actor_start() {
+ let fx = fixture();
+ let reservation = fx
+ .service
+ .reserve_attach(fx.session)
+ .await
+ .expect("attach reserves before reading");
+ let session = fx.repo.get(fx.session).await.expect("session exists");
+ let (stopped, marker) = fx.service.begin_stop(fx.session, false);
+
+ let result = fx
+ .service
+ .activate_reserved(
+ session,
+ RuntimeAttachment::solo(PendingTransport),
+ reservation,
+ )
+ .await;
+ AgentSessionServiceImpl::<
+ InMemoryAgentSessionRepo,
+ FoldedMessageService<InMemoryAgentSessionRepo>,
+ NoOpRealtime,
+ >::wait_stopped(stopped)
+ .await;
+
+ assert!(matches!(result, Err(AgentSessionError::Disconnected(id)) if id == fx.session));
+ assert!(
+ fx.service
+ .active
+ .get(&fx.session)
+ .unwrap()
+ .commands
+ .is_none()
+ );
+ fx.service.active.remove_if(&fx.session, |_, active| {
+ Arc::ptr_eq(&active.marker, &marker)
+ });
+}
+
+#[tokio::test]
+async fn shutdown_prevents_a_reserved_attach_from_spawning() {
+ let fx = fixture();
+ let reservation = fx
+ .service
+ .reserve_attach(fx.session)
+ .await
+ .expect("attach reserves before reading");
+ let session = fx.repo.get(fx.session).await.expect("session exists");
+
+ fx.service.shutdown().await;
+ let result = fx
+ .service
+ .activate_reserved(
+ session,
+ RuntimeAttachment::solo(PendingTransport),
+ reservation,
+ )
+ .await;
+
+ assert!(matches!(result, Err(AgentSessionError::Disconnected(id)) if id == fx.session));
+ assert!(fx.service.tasks.is_closed());
+}
+
+#[tokio::test]
+async fn close_does_not_remove_a_concurrent_delete_guard() {
+ let fx = fixture();
+ let (_stopped, marker) = fx.service.begin_stop(fx.session, true);
+
+ fx.service
+ .close_session(fx.session)
+ .await
+ .expect("close observes the stopped actor");
+
+ assert!(fx.service.active.contains_key(&fx.session));
+ fx.service.active.remove_if(&fx.session, |_, active| {
+ Arc::ptr_eq(&active.marker, &marker)
+ });
+}
+
+#[tokio::test]
+async fn cancellation_does_not_drop_an_effect_batch_after_machine_mutation() {
+ let repo = InMemoryAgentSessionRepo::new();
+ let session = test_session();
+ repo.insert_session(test_agent_session(session));
+ let entered = Arc::new(Notify::new());
+ let release = Arc::new(Notify::new());
+ let logs = BlockingPromptLogs {
+ repo: repo.clone(),
+ entered: entered.clone(),
+ release: release.clone(),
+ hang_disconnect: false,
+ };
+ let (outbound_tx, mut outbound_rx) = mpsc::channel(8);
+ let (inbound_tx, inbound_rx) = mpsc::channel(8);
+ let (commands, command_rx) = mpsc::channel(8);
+ let (handshake, _) = watch::channel(HandshakeStatus::Pending);
+ let actor = SessionActor::new(
+ session,
+ None,
+ "/workspace".to_owned(),
+ RecordingTransport {
+ outbound: outbound_tx,
+ inbound: inbound_rx,
+ },
+ logs,
+ command_rx,
+ handshake,
+ );
+ let active = Arc::new(ActiveSessions::new());
+ let cancellation = CancellationToken::new();
+ let marker = Arc::new(());
+ let (stopped_tx, _) = watch::channel(false);
+ let task = tokio::spawn(run_session(
+ actor,
+ Arc::downgrade(&active),
+ marker,
+ stopped_tx,
+ cancellation.clone(),
+ ));
+
+ open_test_session(&inbound_tx, &mut outbound_rx, session).await;
+
+ let (completed, result) = oneshot::channel();
+ commands
+ .send(SessionCommand {
+ user_id: None,
+ action: AgentAction::prompt("keep dispatching"),
+ action_id: AgentActionId::mint(),
+ completed,
+ span: tracing::info_span!("test.command"),
+ })
+ .await
+ .unwrap();
+ entered.notified().await;
+ cancellation.cancel();
+ release.notify_one();
+
+ let prompt = outbound_rx
+ .recv()
+ .await
+ .expect("prompt is still dispatched");
+ assert!(matches!(
+ prompt,
+ ToRuntimeMessage::Acp(AcpMessage(
+ agent_client_protocol::RawJsonRpcMessage::Request(request)
+ )) if request.method.as_ref() == "session/prompt"
+ ));
+ result.await.unwrap().expect("delivery completes");
+ task.await.unwrap();
+}
+
+#[tokio::test(start_paused = true)]
+async fn live_inbound_logs_do_not_reuse_the_expired_handshake_deadline() {
+ let repo = InMemoryAgentSessionRepo::new();
+ let session = test_session();
+ repo.insert_session(test_agent_session(session));
+ let entered = Arc::new(Notify::new());
+ let release = Arc::new(Notify::new());
+ let logs = BlockingPromptLogs {
+ repo: repo.clone(),
+ entered: entered.clone(),
+ release: release.clone(),
+ hang_disconnect: false,
+ };
+ let (outbound_tx, mut outbound_rx) = mpsc::channel(8);
+ let (inbound_tx, inbound_rx) = mpsc::channel(8);
+ let (commands, command_rx) = mpsc::channel(8);
+ let (handshake, _) = watch::channel(HandshakeStatus::Pending);
+ let actor = SessionActor::new(
+ session,
+ None,
+ "/workspace".to_owned(),
+ RecordingTransport {
+ outbound: outbound_tx,
+ inbound: inbound_rx,
+ },
+ logs,
+ command_rx,
+ handshake,
+ );
+ let active = Arc::new(ActiveSessions::new());
+ let cancellation = CancellationToken::new();
+ let (stopped_tx, _) = watch::channel(false);
+ let task = tokio::spawn(
+ run_session(
+ actor,
+ Arc::downgrade(&active),
+ Arc::new(()),
+ stopped_tx,
+ cancellation.clone(),
+ )
+ .with_current_subscriber(),
+ );
+ open_test_session(&inbound_tx, &mut outbound_rx, session).await;
+
+ release.notify_one();
+ let (completed, result) = oneshot::channel();
+ commands
+ .send(SessionCommand {
+ user_id: None,
+ action: AgentAction::prompt("keep working"),
+ action_id: AgentActionId::mint(),
+ completed,
+ span: tracing::info_span!("test.command"),
+ })
+ .await
+ .unwrap();
+ entered.notified().await;
+ let _prompt = outbound_rx.recv().await.expect("prompt request");
+ result.await.unwrap().expect("prompt delivered");
+
+ tokio::time::advance(std::time::Duration::from_secs(61)).await;
+ inbound_tx
+ .send(ToServerMessage::Acp(AcpMessage(
+ agent_client_protocol::RawJsonRpcMessage::notification(
+ "session/update".to_owned(),
+ serde_json::json!({ "sessionId": "acp-1", "update": {} }),
+ )
+ .unwrap(),
+ )))
+ .await
+ .unwrap();
+
+ let mut persisted = false;
+ for _ in 0..20 {
+ tokio::task::yield_now().await;
+ persisted = AgentSessionLogRepo::list_by_session(&repo, session)
+ .await
+ .unwrap()
+ .iter()
+ .any(|stored| {
+ matches!(
+ &stored.entry.content,
+ Message::ToServer(ToServerMessage::Acp(AcpMessage(
+ agent_client_protocol::RawJsonRpcMessage::Notification(notification)
+ ))) if notification.method.as_ref() == "session/update"
+ )
+ });
+ if persisted {
+ break;
+ }
+ }
+ assert!(persisted, "live update should use the regular log timeout");
+
+ cancellation.cancel();
+ task.await.unwrap();
+}
+
/// Any protocol frame will do: the service only stores it, turn detection is
/// the fold's answer.
fn any_event(session: AgentSessionId) -> AgentSessionLog {
@@ -346,6 +826,29 @@ async fn marking_disconnected_persists_and_publishes_the_event() {
assert_eq!(realtime.published().len(), 1);
}
+#[tokio::test(start_paused = true)]
+async fn marking_disconnected_is_bounded_when_persistence_hangs() {
+ let repo = InMemoryAgentSessionRepo::new();
+ let session = test_session();
+ repo.insert_session(test_agent_session(session));
+ let hanging = BlockingPromptLogs {
+ repo: repo.clone(),
+ entered: Arc::new(Notify::new()),
+ release: Arc::new(Notify::new()),
+ hang_disconnect: true,
+ };
+ let service =
+ AgentSessionServiceImpl::new(hanging, FoldedMessageService::new(repo), NoOpRealtime);
+ let disconnect = tokio::spawn(async move { service.mark_disconnected(session).await });
+ tokio::task::yield_now().await;
+ tokio::time::advance(SESSION_PERSIST_TIMEOUT).await;
+
+ assert!(matches!(
+ disconnect.await.unwrap(),
+ Err(AgentSessionError::LogTimedOut(id)) if id == session
+ ));
+}
+
/// The point of the rework: a connection folds its session once, when it
/// starts, and every frame after that is folded into the state it kept.
///
diff --git a/crates/agent_session/src/domain/session/actors.rs b/crates/agent_session/src/domain/session/actors.rs
index 41e49d5c46..d26df5f32f 100644
--- a/crates/agent_session/src/domain/session/actors.rs
+++ b/crates/agent_session/src/domain/session/actors.rs
@@ -9,25 +9,34 @@
//! not sent.
//!
//! Stepping is the caller's choice: production spawns a "step until stopped"
-//! loop (see [`super::super::service`]), while a test can construct an actor
-//! and crank it by hand - after every [`SessionActor::step`] the world has
-//! fully caught up with that input.
+//! loop (see [`super::super::service`]), while tests exercise its input and
+//! dispatch halves directly.
-use std::collections::VecDeque;
+use std::{collections::VecDeque, time::Duration};
+use agent_client_protocol::RawJsonRpcMessage;
use agent_client_protocol::schema::v1::SessionId;
use agent_runtime_protocol::domain::action::{AgentAction, AgentActionId};
-use agent_runtime_protocol::domain::schema::v0::{SystemEvent, ToRuntimeMessage, ToServerMessage};
+use agent_runtime_protocol::domain::schema::v0::{
+ AcpMessage, SystemEvent, ToRuntimeMessage, ToServerMessage,
+};
use macro_user_id::user_id::MacroUserIdStr;
use tokio::sync::{mpsc, oneshot, watch};
+use tokio::time::Instant;
+use tracing::Instrument as _;
-use crate::domain::error::Result;
+use crate::domain::error::{AgentSessionError, Result};
use crate::domain::model::{AgentSessionId, AgentSessionLog, Message};
use agent_runtime_protocol::domain::ports::{TransportReceiver, TransportSender};
use crate::domain::ports::{AgentConnector, AgentSessionLogWriter, AgentSessionRepo};
-use super::{CloseReason, Effect, HandshakeStatus, Input, SessionMachine};
+use super::{CloseReason, Effect, HandshakeStatus, Input, RuntimeStatus, SessionMachine};
+
+/// How long the ACP handshake has to finish before the session is declared dead.
+const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(60);
+/// How long one caller's action has to reach the runtime.
+const COMMAND_DELIVERY_TIMEOUT: Duration = Duration::from_secs(60);
/// A caller's request to deliver one action, and the wire back to them.
pub(crate) struct SessionCommand {
@@ -35,6 +44,12 @@ pub(crate) struct SessionCommand {
pub(crate) action: AgentAction,
pub(crate) action_id: AgentActionId,
pub(crate) completed: oneshot::Sender<Result<()>>,
+ pub(crate) span: tracing::Span,
+}
+
+pub(crate) struct SessionCompletion {
+ completed: oneshot::Sender<Result<()>>,
+ span: tracing::Span,
}
/// Whether a [`SessionActor`] has more steps to take.
@@ -49,7 +64,7 @@ pub(crate) enum Stepped {
/// One session connection's imperative shell: pulls one input, runs the
/// machine, and executes the effects - nothing else.
pub(crate) struct SessionActor<Connector: AgentConnector, Logs> {
- machine: SessionMachine<oneshot::Sender<Result<()>>>,
+ machine: SessionMachine<SessionCompletion>,
/// The carrier's halves. Sending is shared with whoever else is on the
/// same connection; receiving is this actor's alone, which is why it can
/// be read with `&mut` and needs no lock.
@@ -61,6 +76,10 @@ pub(crate) struct SessionActor<Connector: AgentConnector, Logs> {
/// the handshake, watched for when another session runs it.
handshake: watch::Sender<HandshakeStatus>,
handshake_seen: watch::Receiver<HandshakeStatus>,
+ /// The span covering the handshake, dropped once the runtime is live or
+ /// dead. Every effect run before then hangs off it.
+ handshake_span: Option<tracing::Span>,
+ handshake_deadline: Instant,
}
impl<Connector, Logs> SessionActor<Connector, Logs>
@@ -84,6 +103,12 @@ where
handshake_seen.mark_changed();
let (outbound, inbound) = connector.split();
+ let handshake_span = tracing::info_span!(
+ "agent.acp.handshake",
+ agent.session.id = %id,
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
Self {
outbound,
@@ -96,6 +121,8 @@ where
},
logs,
commands,
+ handshake_span: Some(handshake_span),
+ handshake_deadline: Instant::now() + HANDSHAKE_TIMEOUT,
}
}
@@ -110,15 +137,30 @@ where
self.commands.close();
}
- /// Wait for one input and execute every effect it produces.
- pub(crate) async fn step(&mut self) -> Stepped {
- let input = tokio::select! {
+ /// Wait for the next input without mutating protocol state. Keeping this
+ /// separate from dispatch lets shutdown cancel the wait, never an effect
+ /// batch after the machine has already advanced.
+ pub(crate) async fn next_input(&mut self) -> Input<SessionCompletion> {
+ loop {
+ let handshake_deadline = self.handshake_deadline;
+ let handshake_timeout = async {
+ if matches!(
+ self.machine.status(),
+ RuntimeStatus::Booting | RuntimeStatus::Handshaking
+ ) {
+ tokio::time::sleep_until(handshake_deadline).await;
+ } else {
+ std::future::pending::<()>().await;
+ }
+ };
+ let input = tokio::select! {
+ () = handshake_timeout => Input::Closed(CloseReason::HandshakeTimedOut),
command = self.commands.recv() => match command {
- Some(SessionCommand { user_id, action, action_id, completed }) => Input::Command {
+ Some(SessionCommand { user_id, action, action_id, completed, span }) => Input::Command {
from: user_id,
action,
action_id,
- token: completed,
+ token: SessionCompletion { completed, span },
},
// The service dropped every handle; nobody can reach us.
None => Input::Closed(CloseReason::Abandoned),
@@ -129,7 +171,7 @@ where
Ok(()) = self.handshake_seen.changed() => match *self.handshake_seen.borrow_and_update() {
HandshakeStatus::Ready(restore) => Input::Ready { restore },
// Nothing to act on yet, but the wait must resume.
- HandshakeStatus::Pending | HandshakeStatus::InFlight => return Stepped::Continue,
+ HandshakeStatus::Pending | HandshakeStatus::InFlight => continue,
},
inbound = self.inbound.recv() => match inbound {
Ok(Some(message)) => Input::Inbound(message),
@@ -139,48 +181,137 @@ where
Input::Closed(CloseReason::TransportFailed)
}
},
- };
-
- self.dispatch(input).await
+ };
+ return input;
+ }
}
/// Run the machine and execute its effects in order. An effect that fails
/// to execute voids the rest of its batch and feeds [`Input::Closed`]
/// back in - so a `Complete` never fires for an action that was not sent,
/// and the machine (not this loop) decides what failure means.
- async fn dispatch(&mut self, input: Input<oneshot::Sender<Result<()>>>) -> Stepped {
- let mut effects = VecDeque::from(self.machine.handle(input));
+ pub(crate) async fn dispatch(&mut self, input: Input<SessionCompletion>) -> Stepped {
+ let handshake_deadline = self.handshake_deadline;
+ let mut handshake_span = self.handshake_span.clone();
+ let produced = match &handshake_span {
+ Some(span) => span.in_scope(|| self.machine.handle(input)),
+ None => self.machine.handle(input),
+ };
+ let mut effects = VecDeque::from(produced);
+ self.finish_handshake_if_complete();
+ if self.handshake_span.is_none() {
+ handshake_span = None;
+ }
let mut stepped = Stepped::Continue;
while let Some(effect) = effects.pop_front() {
match effect {
Effect::Send { from, message } => {
- if let Err(error) = self.deliver(from, message).await {
+ let command_span = effects.front().and_then(|effect| match effect {
+ Effect::Complete { token, .. } => Some(token.span.clone()),
+ _ => None,
+ });
+ let delivery = self.deliver(from, message);
+ let (result, close_reason) =
+ match (command_span.as_ref(), handshake_span.as_ref()) {
+ (Some(span), _) => match tokio::time::timeout(
+ COMMAND_DELIVERY_TIMEOUT,
+ delivery.instrument(span.clone()),
+ )
+ .await
+ {
+ Ok(result) => (result, CloseReason::SendFailed),
+ Err(_) => (
+ Err(AgentSessionError::DeliveryTimedOut(self.machine.id())),
+ CloseReason::SendFailed,
+ ),
+ },
+ (None, Some(span)) => match tokio::time::timeout_at(
+ handshake_deadline,
+ delivery.instrument(span.clone()),
+ )
+ .await
+ {
+ Ok(result) => (result, CloseReason::SendFailed),
+ Err(_) => (
+ Err(AgentSessionError::Handshake(format!(
+ "timed out after {} seconds",
+ HANDSHAKE_TIMEOUT.as_secs()
+ ))),
+ CloseReason::HandshakeTimedOut,
+ ),
+ },
+ (None, None) => {
+ match tokio::time::timeout(COMMAND_DELIVERY_TIMEOUT, delivery).await
+ {
+ Ok(result) => (result, CloseReason::SendFailed),
+ Err(_) => (
+ Err(AgentSessionError::DeliveryTimedOut(self.machine.id())),
+ CloseReason::SendFailed,
+ ),
+ }
+ }
+ };
+ if let Err(error) = result {
+ if let Some(span) = command_span {
+ span.record("otel.status_code", "ERROR");
+ span.record("otel.status_description", tracing::field::display(&error));
+ }
tracing::error!(
error = ?error,
id = %self.machine.id(),
"agent session failed to deliver an action"
);
- if let Some(Effect::Complete { token, .. }) = effects.pop_front() {
- let _ = token.send(Err(error));
- }
- effects.clear();
- effects.extend(self.machine.handle(Input::Closed(CloseReason::SendFailed)));
+ self.fail_remaining_completions(&mut effects, error);
+ effects.extend(self.machine.handle(Input::Closed(close_reason)));
+ self.finish_handshake_if_complete();
}
}
Effect::Log { message } => {
- if let Err(error) = self.log(None, Message::ToServer(message)).await {
+ let log = self.log(None, Message::ToServer(message));
+ let (result, close_reason) = match handshake_span.as_ref() {
+ Some(span) => match tokio::time::timeout_at(
+ handshake_deadline,
+ log.instrument(span.clone()),
+ )
+ .await
+ {
+ Ok(result) => (result, CloseReason::LogFailed),
+ Err(_) => (
+ Err(AgentSessionError::Handshake(format!(
+ "timed out after {} seconds",
+ HANDSHAKE_TIMEOUT.as_secs()
+ ))),
+ CloseReason::HandshakeTimedOut,
+ ),
+ },
+ None => match tokio::time::timeout(COMMAND_DELIVERY_TIMEOUT, log).await {
+ Ok(result) => (result, CloseReason::LogFailed),
+ Err(_) => (
+ Err(AgentSessionError::LogTimedOut(self.machine.id())),
+ CloseReason::LogFailed,
+ ),
+ },
+ };
+ if let Err(error) = result {
tracing::error!(
error = ?error,
id = %self.machine.id(),
"agent session failed to persist an inbound message"
);
- effects.clear();
- effects.extend(self.machine.handle(Input::Closed(CloseReason::LogFailed)));
+ self.fail_remaining_completions(&mut effects, error);
+ effects.extend(self.machine.handle(Input::Closed(close_reason)));
+ self.finish_handshake_if_complete();
}
}
Effect::PersistAcpSession { session_id } => {
- if let Err(error) = self.persist_acp_session(session_id).await {
+ let result = tokio::time::timeout(
+ COMMAND_DELIVERY_TIMEOUT,
+ self.persist_acp_session(session_id),
+ )
+ .await
+ .unwrap_or_else(|_| Err(AgentSessionError::LogTimedOut(self.machine.id())));
+ if let Err(error) = result {
tracing::error!(
error = ?error,
id = %self.machine.id(),
@@ -197,24 +328,38 @@ where
let _ = self.handshake.send(HandshakeStatus::Ready(restore));
}
Effect::Complete { token, result } => {
- let _ = token.send(result);
+ if let Err(error) = &result {
+ token.span.record("otel.status_code", "ERROR");
+ token
+ .span
+ .record("otel.status_description", tracing::field::display(error));
+ }
+ let _ = token.completed.send(result);
}
Effect::Stop { reason } => {
- self.log(
+ let terminal_log = self.log(
None,
Message::ToServer(ToServerMessage::Event {
event: SystemEvent::Disconnected,
}),
- )
- .await
- .inspect_err(|error| {
- tracing::error!(
- error = ?error,
- id = %self.machine.id(),
- "agent session failed to persist its disconnect"
- );
- })
- .ok();
+ );
+ let result = tokio::time::timeout(COMMAND_DELIVERY_TIMEOUT, terminal_log).await;
+ match result {
+ Ok(Ok(())) => {}
+ Ok(Err(error)) => {
+ tracing::error!(
+ error = ?error,
+ id = %self.machine.id(),
+ "agent session failed to persist its disconnect"
+ );
+ }
+ Err(_) => {
+ tracing::error!(
+ id = %self.machine.id(),
+ "agent session timed out persisting its disconnect"
+ );
+ }
+ }
tracing::info!(id = %self.machine.id(), %reason, "agent session stopped");
stepped = Stepped::Stopped;
}
@@ -226,11 +371,25 @@ where
/// Log then send: the log entry is written first so the session's history
/// never lacks a message its agent received.
+ #[tracing::instrument(
+ name = "agent.acp.deliver",
+ err,
+ skip(self, from, message),
+ fields(
+ agent.session.id = %self.machine.id(),
+ rpc.system.name = tracing::field::Empty,
+ rpc.method = tracing::field::Empty,
+ )
+ )]
async fn deliver(
&mut self,
from: Option<MacroUserIdStr<'static>>,
message: ToRuntimeMessage,
) -> Result<()> {
+ if let Some(method) = acp_method(&message) {
+ tracing::Span::current().record("rpc.system.name", "jsonrpc");
+ tracing::Span::current().record("rpc.method", method);
+ }
self.log(from, Message::ToRuntime(message.clone())).await?;
self.outbound.send(message).await?;
Ok(())
@@ -255,4 +414,63 @@ where
.set_acp_session_id(self.machine.id(), acp_session_id)
.await
}
+
+ fn finish_handshake_if_complete(&mut self) {
+ match self.machine.status() {
+ RuntimeStatus::Live { .. } => {
+ self.handshake_span.take();
+ }
+ RuntimeStatus::Dead => {
+ if let Some(span) = self.handshake_span.take() {
+ span.record("otel.status_code", "ERROR");
+ span.record("otel.status_description", "ACP handshake failed");
+ }
+ }
+ RuntimeStatus::Booting | RuntimeStatus::Handshaking => {}
+ }
+ }
+
+ /// The failing effect's own caller gets `cause` itself; anything still
+ /// queued behind it never reached the runtime, so it gets `Disconnected`.
+ fn fail_remaining_completions(
+ &self,
+ effects: &mut VecDeque<Effect<SessionCompletion>>,
+ cause: AgentSessionError,
+ ) {
+ let mut stop = None;
+ let mut cause = Some(cause);
+ while let Some(effect) = effects.pop_front() {
+ match effect {
+ Effect::Complete { token, .. } => {
+ let error = cause
+ .take()
+ .unwrap_or_else(|| AgentSessionError::Disconnected(self.machine.id()));
+ token.span.record("otel.status_code", "ERROR");
+ token
+ .span
+ .record("otel.status_description", tracing::field::display(&error));
+ let _ = token.completed.send(Err(error));
+ }
+ effect @ Effect::Stop { .. } => stop = Some(effect),
+ Effect::Send { .. }
+ | Effect::Log { .. }
+ | Effect::PersistAcpSession { .. }
+ | Effect::Initialized { .. } => {}
+ }
+ }
+ if let Some(stop) = stop {
+ effects.push_back(stop);
+ }
+ }
+}
+
+fn acp_method(message: &ToRuntimeMessage) -> Option<&str> {
+ let ToRuntimeMessage::Acp(AcpMessage(frame)) = message else {
+ return None;
+ };
+ match frame {
+ RawJsonRpcMessage::Request(request) => Some(request.method.as_ref()),
+ RawJsonRpcMessage::Notification(notification) => Some(notification.method.as_ref()),
+ RawJsonRpcMessage::Response(_) => None,
+ }
}
diff --git a/crates/agent_session/src/domain/session/tests.rs b/crates/agent_session/src/domain/session/tests.rs
index 6995026be5..0a28dc0546 100644
--- a/crates/agent_session/src/domain/session/tests.rs
+++ b/crates/agent_session/src/domain/session/tests.rs
@@ -110,6 +110,13 @@ fn sent_methods(effects: &[Effect<u32>]) -> Vec<String> {
message: ToRuntimeMessage::Acp(AcpMessage(RawJsonRpcMessage::Request(request))),
..
} => Some(request.method.to_string()),
+ Effect::Send {
+ message:
+ ToRuntimeMessage::Acp(AcpMessage(RawJsonRpcMessage::Notification(
+ notification,
+ ))),
+ ..
+ } => Some(notification.method.to_string()),
_ => None,
})
.collect()
@@ -192,7 +199,7 @@ fn a_second_acp_ready_only_logs() {
}
#[test]
-fn session_new_success_flushes_the_queue_positionally() {
+fn session_new_success_flushes_the_queue() {
let mut machine = machine();
machine.handle(command("first", 1));
machine.handle(command("second", 2));
@@ -200,8 +207,6 @@ fn session_new_success_flushes_the_queue_positionally() {
let effects = machine.handle(session_opened("acp-42"));
- // Each action's completion directly follows its send: delivery is
- // positional, not counted.
assert!(matches!(
effects[..],
[
diff --git a/crates/agent_session/src/domain/session/types.rs b/crates/agent_session/src/domain/session/types.rs
index 3492cc4595..179376688b 100644
--- a/crates/agent_session/src/domain/session/types.rs
+++ b/crates/agent_session/src/domain/session/types.rs
@@ -121,6 +121,8 @@ pub enum CloseReason {
TransportFailed,
/// The shell could not put an action on the transport.
SendFailed,
+ /// The runtime did not finish its ACP handshake before the deadline.
+ HandshakeTimedOut,
/// The shell could not persist a log entry, and an unlogged session may
/// not keep running: the log stream is the session's history.
LogFailed,
@@ -133,6 +135,7 @@ impl std::fmt::Display for CloseReason {
Self::TransportClosed => "the transport closed",
Self::TransportFailed => "the transport failed",
Self::SendFailed => "an action could not be sent",
+ Self::HandshakeTimedOut => "the ACP handshake timed out",
Self::LogFailed => "a log entry could not be persisted",
})
}
diff --git a/crates/agent_trigger/Cargo.toml b/crates/agent_trigger/Cargo.toml
index bf307d89e2..e050c19947 100644
--- a/crates/agent_trigger/Cargo.toml
+++ b/crates/agent_trigger/Cargo.toml
@@ -23,6 +23,7 @@ strum = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
+tokio = { workspace = true, features = ["rt"] }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
diff --git a/crates/agent_trigger/src/domain/mod.rs b/crates/agent_trigger/src/domain/mod.rs
index 4d7ef959d1..987a7eaf1e 100644
--- a/crates/agent_trigger/src/domain/mod.rs
+++ b/crates/agent_trigger/src/domain/mod.rs
@@ -1,3 +1,4 @@
pub mod broker_events;
+pub mod processing;
pub mod service;
pub mod yield_event;
diff --git a/crates/agent_trigger/src/domain/processing.rs b/crates/agent_trigger/src/domain/processing.rs
new file mode 100644
index 0000000000..119ce6a581
--- /dev/null
+++ b/crates/agent_trigger/src/domain/processing.rs
@@ -0,0 +1,67 @@
+//! Shared channel-event processing for agent trigger consumers.
+
+use agent_session::domain::error::AgentSessionError;
+use agent_session::domain::ports::AgentSessionRepo;
+use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};
+use macro_event_broker::{EventBrokerError, MacroEvent as _, MacroEventBroker};
+
+use super::broker_events::AgentTriggerEventName;
+use super::service::{AgentBotLookup, AgentTriggerService};
+
+/// Failure while evaluating or publishing one channel event.
+#[derive(Debug, thiserror::Error)]
+pub enum ProcessChannelEventError {
+ /// Trigger evaluation could not read its session or bot context.
+ #[error(transparent)]
+ Evaluate(#[from] AgentSessionError),
+ /// A yielded event could not be queued for publication.
+ #[error(transparent)]
+ Publish(#[from] EventBrokerError),
+ /// The publication task stopped before reporting its result.
+ #[error("agent event publication task failed")]
+ PublishTask(#[source] tokio::task::JoinError),
+}
+
+/// Evaluate and publish all agent triggers yielded by one channel event.
+///
+/// Transport adapters retain ownership of decode and offset commit so their
+/// `kafka.process` span can cover the complete record lifecycle.
+pub async fn process_channel_event<Repo, Bots, Broker>(
+ trigger: &AgentTriggerService<Repo, Bots>,
+ publisher: &Broker,
+ event: &ChannelMacroEvent,
+) -> Result<(), ProcessChannelEventError>
+where
+ Repo: AgentSessionRepo,
+ Bots: AgentBotLookup,
+ Broker: MacroEventBroker,
+{
+ let ChannelTopicEvent::MessagePosted(posted) = &event.event().event else {
+ return Ok(());
+ };
+ tracing::Span::current().record("macro.event.type", "channel.message_posted");
+
+ let yielded_events = trigger.evaluate(posted).await?;
+ tracing::info!(
+ message_id = %posted.message_id,
+ yielded_count = yielded_events.len(),
+ "agent trigger evaluated channel message"
+ );
+ if yielded_events.is_empty() {
+ tracing::debug!(message_id = %posted.message_id, "agent trigger yielded no event");
+ }
+ for yielded in yielded_events {
+ let event_type: &'static str = AgentTriggerEventName::from(&yielded.event().event).into();
+ tracing::info!(
+ macro.event.id = %yielded.event().event_id,
+ macro.event.type = event_type,
+ "agent trigger yielded event"
+ );
+ publisher
+ .send_event(&yielded)?
+ .await
+ .map_err(ProcessChannelEventError::PublishTask)??;
+ }
+
+ Ok(())
+}
diff --git a/crates/agent_trigger/src/domain/service.rs b/crates/agent_trigger/src/domain/service.rs
index 63a0d45c55..bb67a01cb0 100644
--- a/crates/agent_trigger/src/domain/service.rs
+++ b/crates/agent_trigger/src/domain/service.rs
@@ -45,6 +45,8 @@ where
channel_id = %posted.channel_id,
message_id = %posted.message_id,
thread_id = ?posted.thread_id,
+ channel.message.scope = tracing::field::Empty,
+ agent.mention.bot_count = tracing::field::Empty,
))]
pub async fn evaluate(
&self,
@@ -52,6 +54,15 @@ where
) -> Result<Vec<AgentSessionMacroEvent>> {
let mut mentioned = bot_mention_ids(&posted.mentions);
mentioned.sort_by_key(ToString::to_string);
+ tracing::Span::current().record(
+ "channel.message.scope",
+ if posted.thread_id.is_some() {
+ "thread"
+ } else {
+ "channel_top_level"
+ },
+ );
+ tracing::Span::current().record("agent.mention.bot_count", mentioned.len());
let mut seen_sessions = HashSet::new();
let mut events = Vec::new();
@@ -74,6 +85,16 @@ where
Ok(events)
}
+ #[tracing::instrument(
+ err,
+ skip(self, posted, seen_sessions),
+ fields(
+ channel_id = %posted.channel_id,
+ message_id = %posted.message_id,
+ bot_id = ?mentioned_bot,
+ agent.trigger.outcome = tracing::field::Empty,
+ )
+ )]
async fn evaluate_bot(
&self,
posted: &ChannelMessagePostedMetadata,
@@ -87,11 +108,9 @@ where
if let Some(session_id) = session_id(&existing)
&& !seen_sessions.insert(session_id)
{
- log_no_event(
- posted,
- mentioned_bot,
- NoEventReason::DuplicateSession { session_id },
- );
+ let reason = NoEventReason::DuplicateSession { session_id };
+ tracing::Span::current().record("agent.trigger.outcome", reason.as_ref());
+ log_no_event(posted, mentioned_bot, reason);
return Ok(None);
}
let bot = match &existing {
@@ -109,8 +128,16 @@ where
mentioned_bot,
};
match yield_event(&message, has_agent) {
- AgentSessionEventDecision::Event(event) => Ok(Some(event)),
+ AgentSessionEventDecision::Event(event) => {
+ let outcome = match existing {
+ ChannelSession::None => "top_level_mentioned",
+ ChannelSession::CreatedFromThread(_) => "mention_thread",
+ };
+ tracing::Span::current().record("agent.trigger.outcome", outcome);
+ Ok(Some(event))
+ }
AgentSessionEventDecision::NoEvent(reason) => {
+ tracing::Span::current().record("agent.trigger.outcome", reason.as_ref());
log_no_event(posted, mentioned_bot, reason);
Ok(None)
}
diff --git a/crates/agent_trigger/src/domain/yield_event.rs b/crates/agent_trigger/src/domain/yield_event.rs
index b2bd9cc2a4..649cd79b9e 100644
--- a/crates/agent_trigger/src/domain/yield_event.rs
+++ b/crates/agent_trigger/src/domain/yield_event.rs
@@ -11,7 +11,8 @@ use crate::domain::broker_events::{
};
/// Why evaluating a message did not produce an agent-session event.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::AsRefStr)]
+#[strum(serialize_all = "snake_case")]
pub enum NoEventReason {
/// No existing session and no mentioned bot identified the target agent.
MissingBotContext,
diff --git a/crates/channel_bots/src/domain/trigger_detector/tests.rs b/crates/channel_bots/src/domain/trigger_detector/tests.rs
index b331cce92f..0d7c503498 100644
--- a/crates/channel_bots/src/domain/trigger_detector/tests.rs
+++ b/crates/channel_bots/src/domain/trigger_detector/tests.rs
@@ -162,6 +162,7 @@ fn candidate(
deleted_at: None,
},
mentioned_bot_ids,
+ span: tracing::Span::none(),
}
}
diff --git a/crates/channel_bots/src/inbound/bot_trigger_router.rs b/crates/channel_bots/src/inbound/bot_trigger_router.rs
index 4b6f9947ed..00b10076be 100644
--- a/crates/channel_bots/src/inbound/bot_trigger_router.rs
+++ b/crates/channel_bots/src/inbound/bot_trigger_router.rs
@@ -5,6 +5,7 @@ use std::sync::Arc;
use channels::domain::ports::ChannelService;
use channels::domain::side_effects::ChannelBotTrigger;
use tokio::sync::mpsc::UnboundedReceiver;
+use tracing::Instrument as _;
use crate::domain::{
models::BotEvent,
@@ -61,8 +62,9 @@ where
tokio::spawn(async move {
while let Some(candidate) = candidates.recv().await {
let router = self.clone();
+ let span = candidate.span.clone();
tokio::spawn(async move {
- router.run(candidate).await;
+ router.run(candidate).instrument(span).await;
});
}
});
diff --git a/crates/channels/src/domain/service.rs b/crates/channels/src/domain/service.rs
index 26cf0328d4..b6e8d4bb12 100644
--- a/crates/channels/src/domain/service.rs
+++ b/crates/channels/src/domain/service.rs
@@ -19,6 +19,7 @@ use crate::domain::{
ChannelMessagesErr, ChannelMessagesQueryResult, ChannelMutationErr,
ChannelReferenceSharePermissions, ChannelRepo, ChannelService,
},
+ side_effects::bot_mention_ids,
};
use bot_id::BotIdStr;
use bot_id::cowlike::CowLike;
@@ -591,7 +592,16 @@ where
}
}
- #[tracing::instrument(err, skip(self, req))]
+ #[tracing::instrument(
+ err,
+ skip(self, req),
+ fields(
+ channel.id = %channel_id,
+ channel.message.scope = tracing::field::Empty,
+ channel.message.mention_count = tracing::field::Empty,
+ agent.mention.bot_count = tracing::field::Empty,
+ )
+ )]
async fn post_message(
&self,
actor: Sender,
@@ -605,6 +615,19 @@ where
if actor.as_bot().is_some() && req.mentions.is_empty() {
req.mentions = self.extract_content_mentions(&req.content).await;
}
+ tracing::Span::current().record(
+ "channel.message.scope",
+ if req.thread_id.is_some() {
+ "thread"
+ } else {
+ "channel_top_level"
+ },
+ );
+ tracing::Span::current().record("channel.message.mention_count", req.mentions.len());
+ tracing::Span::current().record(
+ "agent.mention.bot_count",
+ bot_mention_ids(&req.mentions).len(),
+ );
let message = self
.repo
diff --git a/crates/channels/src/domain/side_effects.rs b/crates/channels/src/domain/side_effects.rs
index 3eadb94776..a7781e89ec 100644
--- a/crates/channels/src/domain/side_effects.rs
+++ b/crates/channels/src/domain/side_effects.rs
@@ -25,6 +25,7 @@ use macro_event_broker::{MacroEventBroker, NoopMacroEventBroker};
use macro_user_id::{cowlike::CowLike, user_id::MacroUserIdStr};
use std::collections::HashSet;
use tokio::sync::mpsc::UnboundedSender;
+use tracing::Instrument as _;
use uuid::Uuid;
/// Entity type used by message mentions that target a bot.
@@ -44,6 +45,8 @@ pub struct ChannelBotTrigger {
/// Bots explicitly mentioned in the message that are active in the
/// channel. Empty when the message mentions no bot.
pub mentioned_bot_ids: Vec<BotId>,
+ /// Trace active when the candidate was dispatched.
+ pub span: tracing::Span,
}
/// Sender for bot-trigger candidates derived from channel messages.
@@ -423,6 +426,11 @@ impl<C, R, N, K, B> ChannelSideEffectService<C, R, N, K, B> {
channel_id,
message: message.clone(),
mentioned_bot_ids: active_bot_mention_ids(mentions, participants),
+ span: tracing::info_span!(
+ "channel.bot_trigger",
+ channel.id = %channel_id,
+ channel.message.id = %message.id,
+ ),
})
.is_err()
{
@@ -454,9 +462,12 @@ where
{
fn dispatch(&self, event: ChannelEvent) {
let handler = self.handler.clone();
- tokio::spawn(async move {
- handler.handle(event).await;
- });
+ tokio::spawn(
+ async move {
+ handler.handle(event).await;
+ }
+ .instrument(tracing::info_span!("channel.side_effects")),
+ );
}
}
diff --git a/crates/connection_gateway_client/Cargo.toml b/crates/connection_gateway_client/Cargo.toml
index 2124e819b2..8176dd2c09 100644
--- a/crates/connection_gateway_client/Cargo.toml
+++ b/crates/connection_gateway_client/Cargo.toml
@@ -8,6 +8,7 @@ version = "0.1.0"
anyhow = { workspace = true }
connection_gateway_models = { path = "../connection_gateway_models" }
model-entity = { path = "../model-entity" }
+macro_tower_layers = { path = "../macro_tower_layers" }
reqwest = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
diff --git a/crates/connection_gateway_client/src/client.rs b/crates/connection_gateway_client/src/client.rs
index 8e0536ed74..39168fa475 100644
--- a/crates/connection_gateway_client/src/client.rs
+++ b/crates/connection_gateway_client/src/client.rs
@@ -23,6 +23,12 @@ struct Response {
}
impl ConnectionGatewayClient {
+ fn with_trace_headers(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+ let mut headers = reqwest::header::HeaderMap::new();
+ macro_tower_layers::inject_trace_headers(&mut headers);
+ request.headers(headers)
+ }
+
/// Create a new connection gateway client.
pub fn new(internal_auth_key: String, connection_gateway_url: String) -> Self {
let mut headers = reqwest::header::HeaderMap::new();
@@ -51,12 +57,12 @@ impl ConnectionGatewayClient {
message,
};
+ let request = self.client.post(format!(
+ "{}/message/send/{}/{}",
+ self.connection_gateway_url, entity.entity_type, entity.entity_id
+ ));
let res = self
- .client
- .post(format!(
- "{}/message/send/{}/{}",
- self.connection_gateway_url, entity.entity_type, entity.entity_id
- ))
+ .with_trace_headers(request)
.json(&body)
.send()
.await?
@@ -80,15 +86,11 @@ impl ConnectionGatewayClient {
entities,
message_type,
};
- let res = self
- .client
- .post(format!(
- "{}/message/batch_send",
- self.connection_gateway_url
- ))
- .json(&body)
- .send()
- .await?;
+ let request = self.client.post(format!(
+ "{}/message/batch_send",
+ self.connection_gateway_url
+ ));
+ let res = self.with_trace_headers(request).json(&body).send().await?;
let json = res.json().await?;
@@ -103,15 +105,11 @@ impl ConnectionGatewayClient {
messages: Vec<UniqueMessage>,
) -> anyhow::Result<Vec<MessageReceipt>> {
let body = BatchSendUniqueMessagesBody { messages };
- let res = self
- .client
- .post(format!(
- "{}/message/batch_send_unique",
- self.connection_gateway_url
- ))
- .json(&body)
- .send()
- .await?;
+ let request = self.client.post(format!(
+ "{}/message/batch_send_unique",
+ self.connection_gateway_url
+ ));
+ let res = self.with_trace_headers(request).json(&body).send().await?;
let json = res.json().await?;
@@ -122,14 +120,11 @@ impl ConnectionGatewayClient {
/// Get users who are interacting with a given entity.
pub async fn track_entity_users(&self, entity: Entity<'_>) -> anyhow::Result<Vec<String>> {
- let res = self
- .client
- .get(format!(
- "{}/track/{}/{}",
- self.connection_gateway_url, entity.entity_type, entity.entity_id
- ))
- .send()
- .await?;
+ let request = self.client.get(format!(
+ "{}/track/{}/{}",
+ self.connection_gateway_url, entity.entity_type, entity.entity_id
+ ));
+ let res = self.with_trace_headers(request).send().await?;
let json = res.json().await?;
diff --git a/crates/kafka_util/Cargo.toml b/crates/kafka_util/Cargo.toml
index 882a4f17b9..c828f6b6ab 100644
--- a/crates/kafka_util/Cargo.toml
+++ b/crates/kafka_util/Cargo.toml
@@ -10,9 +10,11 @@ aws-types = { workspace = true }
either = { workspace = true }
macro_env = { path = "../macro_env" }
macro_env_var = { path = "../macro_env_var" }
+opentelemetry = { workspace = true, features = ["trace"] }
rdkafka = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
+tracing-opentelemetry = { workspace = true }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/kafka_util/src/lib.rs b/crates/kafka_util/src/lib.rs
index 0b6ed58282..e7acd2d9d7 100644
--- a/crates/kafka_util/src/lib.rs
+++ b/crates/kafka_util/src/lib.rs
@@ -8,16 +8,19 @@
#[cfg(test)]
mod test;
+use std::collections::HashMap;
use std::marker::PhantomData;
use std::time::Duration;
use either::Either;
use macro_env::Environment;
+use opentelemetry::propagation::{Extractor, Injector};
use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer};
use rdkafka::error::{KafkaError, KafkaResult};
-use rdkafka::message::{BorrowedMessage, Message as _};
+use rdkafka::message::{BorrowedMessage, Header, Headers, Message, OwnedHeaders};
use rdkafka::producer::{FutureProducer, FutureRecord};
use rdkafka::{ClientConfig, Offset, TopicPartitionList};
+use tracing_opentelemetry::OpenTelemetrySpanExt as _;
use uuid::Uuid;
pub use msk_iam::{MskIamClientContext, configure_sasl_iam};
@@ -28,6 +31,97 @@ const UNGROUPED_GROUP_PREFIX: &str = "macro-event-broker-independent";
const MESSAGE_TIMEOUT_MS: &str = "5000";
const SEND_TIMEOUT: Duration = Duration::from_secs(5);
+#[derive(Default)]
+struct TraceHeaderInjector(HashMap<String, String>);
+
+impl Injector for TraceHeaderInjector {
+ fn set(&mut self, key: &str, value: String) {
+ self.0.insert(key.to_owned(), value);
+ }
+}
+
+struct TraceHeaderExtractor<'a, H: Headers>(&'a H);
+
+impl<H: Headers> Extractor for TraceHeaderExtractor<'_, H> {
+ fn get(&self, key: &str) -> Option<&str> {
+ self.0.iter().find_map(|header| {
+ (header.key == key)
+ .then(|| {
+ header
+ .value
+ .and_then(|value| std::str::from_utf8(value).ok())
+ })
+ .flatten()
+ })
+ }
+
+ fn keys(&self) -> Vec<&str> {
+ self.0.iter().map(|header| header.key).collect()
+ }
+}
+
+fn current_trace_headers() -> OwnedHeaders {
+ let context = tracing::Span::current().context();
+ let mut injector = TraceHeaderInjector::default();
+ opentelemetry::global::get_text_map_propagator(|propagator| {
+ propagator.inject_context(&context, &mut injector);
+ });
+
+ injector
+ .0
+ .into_iter()
+ .fold(OwnedHeaders::new(), |headers, (key, value)| {
+ headers.insert(Header {
+ key: &key,
+ value: Some(&value),
+ })
+ })
+}
+
+fn remote_trace_context<M: Message>(message: &M) -> opentelemetry::Context {
+ opentelemetry::global::get_text_map_propagator(|propagator| {
+ message
+ .headers()
+ .map_or_else(opentelemetry::Context::new, |headers| {
+ propagator.extract(&TraceHeaderExtractor(headers))
+ })
+ })
+}
+
+/// Creates a Kafka processing span parented to the message's propagated W3C context.
+///
+/// The caller should instrument the complete decode, handling, and commit future
+/// with this span so the consumer operation represents all record processing.
+pub fn consumer_span<M: Message>(message: &M, consumer_group: &'static str) -> tracing::Span {
+ let span = tracing::info_span!(
+ "kafka.process",
+ otel.kind = "consumer",
+ messaging.system = "kafka",
+ messaging.operation.name = "process",
+ messaging.operation.type = "process",
+ messaging.consumer.group.name = consumer_group,
+ messaging.destination.name = message.topic(),
+ messaging.destination.partition.id = %message.partition(),
+ messaging.kafka.offset = message.offset(),
+ macro.event.id = tracing::field::Empty,
+ macro.event.type = tracing::field::Empty,
+ agent.session.id = tracing::field::Empty,
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ let parent = remote_trace_context(message);
+ // Setting an empty context is intentional: headerless records must not
+ // inherit an unrelated process- or task-lifetime ambient span.
+ let _ = span.set_parent(parent);
+ span
+}
+
+/// Marks a manually-created messaging span as failed.
+pub fn record_span_error(span: &tracing::Span, error: &(impl std::fmt::Display + ?Sized)) {
+ span.record("otel.status_code", "ERROR");
+ span.record("otel.status_description", tracing::field::display(error));
+}
+
/// Failure to construct an environment-specific Kafka consumer.
#[derive(Debug, thiserror::Error)]
pub enum KafkaConsumerError {
@@ -246,9 +340,23 @@ impl KafkaEventProducer {
}
/// Sends a keyed payload to `topic` and waits for delivery confirmation.
- #[tracing::instrument(err, skip(self, payload), fields(topic, key))]
+ #[tracing::instrument(
+ name = "kafka.publish",
+ err,
+ skip(self, payload),
+ fields(
+ otel.kind = "producer",
+ messaging.system = "kafka",
+ messaging.operation.name = "publish",
+ messaging.operation.type = "send",
+ messaging.destination.name = topic,
+ )
+ )]
pub async fn send(&self, topic: &str, key: &str, payload: &[u8]) -> KafkaResult<()> {
- let record = FutureRecord::to(topic).key(key).payload(payload);
+ let record = FutureRecord::to(topic)
+ .key(key)
+ .payload(payload)
+ .headers(current_trace_headers());
either::for_both!(&self.producer.0, producer => producer.send(record, SEND_TIMEOUT).await)
.map(|_| ())
.map_err(|(error, _)| error)
diff --git a/crates/macro_event_broker/src/domain/models.rs b/crates/macro_event_broker/src/domain/models.rs
index 6b1d51b0f8..a5e6141bd8 100644
--- a/crates/macro_event_broker/src/domain/models.rs
+++ b/crates/macro_event_broker/src/domain/models.rs
@@ -181,7 +181,6 @@ impl<T, M> MessageWrapper<T, M> {
impl<T: MessageParts, M: MacroEventCollection> MessageWrapper<T, M> {
/// Decodes the message into one of the declared event types.
- #[tracing::instrument(err, skip(self))]
pub fn decode_payload(&self) -> Result<M, EventBrokerError> {
M::decode(&self.inner)
}
diff --git a/crates/macro_event_broker/src/domain/service.rs b/crates/macro_event_broker/src/domain/service.rs
index a9b5be68dd..6dceaef4d9 100644
--- a/crates/macro_event_broker/src/domain/service.rs
+++ b/crates/macro_event_broker/src/domain/service.rs
@@ -43,7 +43,6 @@ where
/// Receives the next message from the underlying consumer.
///
/// Call [`MessageWrapper::decode_payload`] to decode its declared event.
- #[tracing::instrument(skip(self), err)]
pub async fn recv<'a>(
&'a self,
) -> Result<MessageWrapper<C::MessageType<'a>, M>, rootcause::Report> {
@@ -89,7 +88,6 @@ impl<P: EventPublisher, S: Spawner> MacroEventBrokerService<P, S> {
}
impl<P: EventPublisher, S: Spawner> MacroEventBroker for MacroEventBrokerService<P, S> {
- #[tracing::instrument(err, skip(self, event), fields(topic = event.topic(), key = %event.key()))]
fn send_event<E: MacroEvent + ?Sized>(
&self,
event: &E,
diff --git a/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs
index 90a9a3be95..380669ccc4 100644
--- a/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs
+++ b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs
@@ -68,7 +68,6 @@ impl<T: GroupName, M> KafkaConsumerAdapter<T, M> {
}
/// Commits a message using the caller-selected commit mode.
- #[tracing::instrument(err, skip(self))]
pub fn commit_message(
&self,
message: &BorrowedMessage<'_>,
diff --git a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs
index 432bfbb0df..47a725ec20 100644
--- a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs
+++ b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs
@@ -27,7 +27,6 @@ impl KafkaEventPublisher {
}
impl EventPublisher for KafkaEventPublisher {
- #[tracing::instrument(err, skip(self, payload), fields(topic = T::TOPIC_STR, key = %key))]
async fn publish<T: Topic>(&self, key: &str, payload: &[u8]) -> Result<(), EventBrokerError> {
self.producer
.send(T::TOPIC_STR, key, payload)
diff --git a/crates/macro_tower_layers/src/test.rs b/crates/macro_tower_layers/src/test.rs
index 93bd9483c7..655b5a9ea7 100644
--- a/crates/macro_tower_layers/src/test.rs
+++ b/crates/macro_tower_layers/src/test.rs
@@ -318,7 +318,6 @@ fn inject_trace_headers_emits_current_span_traceparent() {
let mut headers = http::HeaderMap::new();
inject_trace_headers(&mut headers);
-
let traceparent = headers
.get("traceparent")
.expect("traceparent header should be injected inside a span")
diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rs
index 9a5bb64575..09104d0d54 100644
--- a/services/agent_harness_service/src/main.rs
+++ b/services/agent_harness_service/src/main.rs
@@ -11,10 +11,10 @@ mod bots_directory;
mod config;
mod trigger;
-use std::sync::Arc;
+use std::{future::Future, pin::Pin, sync::Arc};
use agent_fold::domain::service::FoldedMessageService;
-use agent_harness::domain::model::SessionDefaults;
+use agent_harness::domain::model::{HarnessCommand, SessionDefaults};
use agent_harness::domain::service::AgentHarnessService;
use agent_harness::inbound::kafka::{RoutedTrigger, route_agent_trigger};
use agent_harness::inbound::runtime_gateway::RuntimeGatewayState;
@@ -47,7 +47,7 @@ use channels::outbound::pg_channels_repo::PgChannelsRepo;
use channels::outbound::pg_side_effect_context::PgChannelSideEffectContext;
use config::{Config, Environment};
use connection_gateway_client::ConnectionGatewayClient;
-use kafka_util::{GroupName, KafkaEventConsumer};
+use kafka_util::{GroupName, KafkaEventConsumer, consumer_span, record_span_error};
use lexical_client::LexicalClient;
use macro_auth::middleware::decode_jwt::JwtValidationArgs;
use macro_authorization::{
@@ -63,6 +63,7 @@ use macro_service_urls::{ConnectionGatewayUrl, LexicalServiceUrl};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
use sqlx::postgres::PgPoolOptions;
+use tracing::Instrument as _;
/// Consumer group owning this harness's agent-session offsets.
///
@@ -87,9 +88,25 @@ fn commit_message(consumer: &HarnessConsumer, message: &BorrowedMessage<'_>) ->
.map_err(|error| anyhow::anyhow!("failed to commit agent session offset: {error:?}"))
}
+type HarnessWork =
+ Pin<Box<dyn Future<Output = agent_harness::domain::error::Result<()>> + Send + 'static>>;
+
+struct PendingHarnessWork {
+ session_id: agent_session::domain::model::AgentSessionId,
+ span: tracing::Span,
+ work: HarnessWork,
+ description: &'static str,
+}
+
#[tokio::main]
async fn main() -> anyhow::Result<()> {
- MacroEntrypoint::default().init();
+ let entrypoint = MacroEntrypoint::default().init();
+ let result = run().await;
+ entrypoint.shutdown();
+ result
+}
+
+async fn run() -> anyhow::Result<()> {
agent_harness::install_tls_provider();
let config = Config::from_env()?;
let bot_id = BotId::new_from_uuid(config.harness_bot_id);
@@ -334,69 +351,115 @@ async fn main() -> anyhow::Result<()> {
continue;
}
};
- let kafka_message = message.inner();
- let event = match message.decode_payload() {
- Ok(DeclaredMacroEvent::AgentSessionMacroEvent(event)) => event,
- Err(error) => {
- tracing::error!(
- error = ?error,
- partition = kafka_message.partition(),
- offset = kafka_message.offset(),
- "dropping undecodable agent session event"
- );
- match commit_message(&consumer, kafka_message) {
- Ok(()) => continue,
- Err(error) => {
- run_error = Some(error);
- break;
- }
+ let span = consumer_span(message.inner(), AgentHarnessConsumerGroup::GROUP_NAME);
+ let processing = async {
+ let kafka_message = message.inner();
+ let event = match message.decode_payload() {
+ Ok(DeclaredMacroEvent::AgentSessionMacroEvent(event)) => event,
+ Err(error) => {
+ record_span_error(&tracing::Span::current(), &error);
+ tracing::error!(
+ error = ?error,
+ partition = kafka_message.partition(),
+ offset = kafka_message.offset(),
+ "dropping undecodable agent session event"
+ );
+ commit_message(&consumer, kafka_message)?;
+ return Ok::<_, anyhow::Error>(None);
}
- }
- };
+ };
+ tracing::Span::current()
+ .record("macro.event.id", tracing::field::display(event.event().event_id));
- let routed = match route_agent_trigger(event.event().event.clone(), bot_id) {
- Ok(routed) => routed,
- Err(skipped) => {
- tracing::debug!(?skipped, "skipped an agent session event");
- match commit_message(&consumer, kafka_message) {
- Ok(()) => continue,
- Err(error) => {
- run_error = Some(error);
- break;
- }
+ let routed = match route_agent_trigger(event.event().event.clone(), bot_id) {
+ Ok(routed) => routed,
+ Err(skipped) => {
+ tracing::debug!(?skipped, "skipped an agent session event");
+ commit_message(&consumer, kafka_message)?;
+ return Ok(None);
}
- }
- };
+ };
- // Intentionally at-most-once: keep ingestion simple and never
- // let concurrent task completion commit Kafka offsets out of order.
- if let Err(error) = commit_message(&consumer, kafka_message) {
- run_error = Some(error);
- break;
- }
- match routed {
- RoutedTrigger::Command(session_id, command) => {
- let execution = harness.execute(session_id, command);
- tasks.spawn(async move {
- match execution.await {
- Ok(()) => tracing::info!(%session_id, "executed an agent harness command"),
- Err(error) => {
- tracing::error!(error = ?error, %session_id, "failed to execute an agent harness command");
+ let pending = match routed {
+ RoutedTrigger::Command(session_id, command) => {
+ tracing::Span::current()
+ .record("agent.session.id", tracing::field::display(session_id));
+ let event_type = match &command {
+ HarnessCommand::Open(_) => "agent_trigger.new",
+ HarnessCommand::Deliver(_) => "agent_trigger.existing",
+ HarnessCommand::Delete => "agent_trigger.delete",
+ HarnessCommand::SetSandboxSize(_) => {
+ "agent_trigger.set_sandbox_size"
}
+ };
+ tracing::Span::current().record("macro.event.type", event_type);
+ let execution_span = tracing::info_span!(
+ "harness.execute",
+ agent.session.id = %session_id,
+ agent.command.type = event_type,
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ // `execute` admits synchronously, so entering here is
+ // what carries this child context through the queue.
+ let execution = execution_span
+ .in_scope(|| harness.execute(session_id, command));
+ PendingHarnessWork {
+ session_id,
+ span: execution_span,
+ work: Box::pin(execution),
+ description: "executed an agent harness command",
}
- });
- }
- RoutedTrigger::Announce(session_id, prompt) => {
- let harness = harness.clone();
- tasks.spawn(async move {
- match harness.announce_external_prompt(session_id, prompt).await {
- Ok(()) => tracing::info!(%session_id, "announced an external prompt"),
- Err(error) => {
- tracing::error!(error = ?error, %session_id, "failed to announce an external prompt");
- }
+ }
+ RoutedTrigger::Announce(session_id, prompt) => {
+ tracing::Span::current()
+ .record("agent.session.id", tracing::field::display(session_id));
+ tracing::Span::current()
+ .record("macro.event.type", "agent_trigger.announce");
+ let execution_span = tracing::info_span!(
+ "harness.announce",
+ agent.session.id = %session_id,
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ let harness = harness.clone();
+ PendingHarnessWork {
+ session_id,
+ span: execution_span,
+ work: Box::pin(async move {
+ harness.announce_external_prompt(session_id, prompt).await
+ }),
+ description: "announced an external prompt",
}
- });
+ }
+ };
+
+ // Intentionally at-most-once: admission precedes commit, but
+ // long-running harness work is independent of Kafka afterward.
+ commit_message(&consumer, kafka_message)?;
+ Ok(Some(pending))
+ }
+ .instrument(span.clone())
+ .await;
+
+ let pending = match processing {
+ Ok(pending) => pending,
+ Err(error) => {
+ record_span_error(&span, &error);
+ run_error = Some(error);
+ break;
}
+ };
+ if let Some(PendingHarnessWork { session_id, span, work, description }) = pending {
+ tasks.spawn(async move {
+ match work.instrument(span.clone()).await {
+ Ok(()) => tracing::info!(%session_id, description, "agent harness work completed"),
+ Err(error) => {
+ record_span_error(&span, &error);
+ tracing::error!(error = ?error, %session_id, "agent harness work failed");
+ }
+ }
+ });
}
}
Some(result) = tasks.join_next(), if !tasks.is_empty() => {
@@ -409,7 +472,10 @@ async fn main() -> anyhow::Result<()> {
http.abort();
trigger.abort();
- container_shutdown.shutdown_all().await;
+ let stop_failures = container_shutdown.shutdown_all().await;
+ if stop_failures > 0 {
+ tracing::error!(stop_failures, "some sandboxes failed to stop");
+ }
while let Some(result) = tasks.join_next().await {
if let Err(error) = result {
@@ -417,11 +483,6 @@ async fn main() -> anyhow::Result<()> {
}
}
- let stop_failures = container_shutdown.shutdown_all().await;
- if stop_failures > 0 {
- tracing::error!(stop_failures, "some sandboxes failed to stop");
- }
-
match run_error {
Some(error) => Err(error),
None => Ok(()),
diff --git a/services/agent_harness_service/src/trigger.rs b/services/agent_harness_service/src/trigger.rs
index d41d4e63f4..eaf9ce9bc3 100644
--- a/services/agent_harness_service/src/trigger.rs
+++ b/services/agent_harness_service/src/trigger.rs
@@ -1,21 +1,22 @@
//! Channel-message consumer that emits agent-session trigger events.
use agent_session::outbound::postgres::PgAgentSessionRepo;
+use agent_trigger::domain::processing::process_channel_event;
use agent_trigger::domain::service::{AgentBotLookup, AgentTriggerService};
-use anyhow::Context as _;
use bot_id::BotId;
use bots::domain::ports::BotRepo as _;
use bots::outbound::pg_bots_repo::PgBotsRepo;
-use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};
-use kafka_util::{GroupName, KafkaEventConsumer};
+use channels::domain::broker_events::ChannelMacroEvent;
+use kafka_util::{GroupName, KafkaEventConsumer, consumer_span, record_span_error};
use macro_event_broker::{
- KafkaConsumerAdapter, KafkaEventPublisher, MacroEvent as _, MacroEventBroker as _,
- MacroEventBrokerService, MacroEventCollection as _, MacroEventConsumerService,
+ KafkaConsumerAdapter, KafkaEventPublisher, MacroEvent as _, MacroEventBrokerService,
+ MacroEventCollection as _, MacroEventConsumerService,
};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
use sqlx::PgPool;
use tokio::time::{Duration, sleep};
+use tracing::Instrument as _;
struct AgentTriggerConsumerGroup;
@@ -87,38 +88,37 @@ async fn run(pool: PgPool, kafka_brokers: String) -> anyhow::Result<()> {
continue;
}
};
- let kafka_message = message.inner();
- let event = match message.decode_payload() {
- Ok(DeclaredChannelEvent::ChannelMacroEvent(event)) => event,
- Err(error) => {
- tracing::error!(
- error = ?error,
- partition = kafka_message.partition(),
- offset = kafka_message.offset(),
- "dropping undecodable channel event"
- );
- commit_message(&consumer, kafka_message)?;
- continue;
- }
- };
+ let span = consumer_span(message.inner(), AgentTriggerConsumerGroup::GROUP_NAME);
+ let result = async {
+ let kafka_message = message.inner();
+ let event = match message.decode_payload() {
+ Ok(DeclaredChannelEvent::ChannelMacroEvent(event)) => event,
+ Err(error) => {
+ record_span_error(&tracing::Span::current(), &error);
+ tracing::error!(
+ error = ?error,
+ partition = kafka_message.partition(),
+ offset = kafka_message.offset(),
+ "dropping undecodable channel event"
+ );
+ commit_message(&consumer, kafka_message)?;
+ return Ok::<(), anyhow::Error>(());
+ }
+ };
+ tracing::Span::current().record(
+ "macro.event.id",
+ tracing::field::display(event.event().event_id),
+ );
- let ChannelTopicEvent::MessagePosted(posted) = &event.event().event else {
+ process_channel_event(&trigger, &publisher, &event).await?;
commit_message(&consumer, kafka_message)?;
- continue;
- };
- let yielded_events = trigger.evaluate(posted).await?;
- if yielded_events.is_empty() {
- tracing::debug!(message_id = %posted.message_id, "agent trigger yielded no event");
+ Ok(())
}
- for yielded in yielded_events {
- let json = serde_json::to_string(yielded.event())?;
- tracing::info!(yielded_event = %json, "agent trigger yielded event");
- let publish = publisher.send_event(&yielded)?;
- publish
- .await
- .context("agent event publication task failed")??;
+ .instrument(span.clone())
+ .await;
+ if let Err(error) = &result {
+ record_span_error(&span, error);
}
-
- commit_message(&consumer, kafka_message)?;
+ result?;
}
}
diff --git a/services/agent_trigger_service/src/main.rs b/services/agent_trigger_service/src/main.rs
index cec054db1e..33c45645d7 100644
--- a/services/agent_trigger_service/src/main.rs
+++ b/services/agent_trigger_service/src/main.rs
@@ -3,22 +3,24 @@
mod config;
use agent_session::outbound::postgres::PgAgentSessionRepo;
+use agent_trigger::domain::processing::process_channel_event;
use agent_trigger::domain::service::{AgentBotLookup, AgentTriggerService};
use anyhow::Context as _;
use bots::domain::models::BotId;
use bots::domain::ports::BotRepo as _;
use bots::outbound::pg_bots_repo::PgBotsRepo;
-use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};
+use channels::domain::broker_events::ChannelMacroEvent;
use config::Config;
-use kafka_util::{GroupName, KafkaEventConsumer};
+use kafka_util::{GroupName, KafkaEventConsumer, consumer_span, record_span_error};
use macro_entrypoint::{MacroEntrypoint, shutdown_signal};
use macro_event_broker::{
- KafkaConsumerAdapter, KafkaEventPublisher, MacroEvent as _, MacroEventBroker as _,
- MacroEventBrokerService, MacroEventCollection as _, MacroEventConsumerService,
+ KafkaConsumerAdapter, KafkaEventPublisher, MacroEvent as _, MacroEventBrokerService,
+ MacroEventCollection as _, MacroEventConsumerService,
};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
use sqlx::postgres::PgPoolOptions;
+use tracing::Instrument as _;
struct AgentTriggerConsumerGroup;
@@ -51,9 +53,14 @@ impl AgentBotLookup for PgAgentBotLookup {
}
#[tokio::main]
-#[tracing::instrument(err)]
async fn main() -> anyhow::Result<()> {
- MacroEntrypoint::default().init();
+ let entrypoint = MacroEntrypoint::default().init();
+ let result = run().await;
+ entrypoint.shutdown();
+ result
+}
+
+async fn run() -> anyhow::Result<()> {
let config = Config::from_env()?;
let pool = PgPoolOptions::new()
.min_connections(1)
@@ -84,6 +91,7 @@ async fn main() -> anyhow::Result<()> {
);
let mut shutdown = std::pin::pin!(shutdown_signal());
+ let mut run_error = None;
loop {
tokio::select! {
() = &mut shutdown => {
@@ -98,40 +106,44 @@ async fn main() -> anyhow::Result<()> {
continue;
}
};
- let kafka_message = message.inner();
- let event = match message.decode_payload() {
- Ok(DeclaredMacroEvent::ChannelMacroEvent(event)) => event,
- Err(error) => {
- tracing::error!(
- error = ?error,
- partition = kafka_message.partition(),
- offset = kafka_message.offset(),
- "dropping undecodable channel event"
- );
- commit_message(&consumer, kafka_message)?;
- continue;
- }
- };
-
- let ChannelTopicEvent::MessagePosted(posted) = &event.event().event else {
+ let span = consumer_span(message.inner(), AgentTriggerConsumerGroup::GROUP_NAME);
+ let result = async {
+ let kafka_message = message.inner();
+ let event = match message.decode_payload() {
+ Ok(DeclaredMacroEvent::ChannelMacroEvent(event)) => event,
+ Err(error) => {
+ record_span_error(&tracing::Span::current(), &error);
+ tracing::error!(
+ error = ?error,
+ partition = kafka_message.partition(),
+ offset = kafka_message.offset(),
+ "dropping undecodable channel event"
+ );
+ commit_message(&consumer, kafka_message)?;
+ return Ok::<(), anyhow::Error>(());
+ }
+ };
+ tracing::Span::current().record("macro.event.id", tracing::field::display(event.event().event_id));
+
+ process_channel_event(&trigger, &publisher, &event).await?;
commit_message(&consumer, kafka_message)?;
- continue;
- };
- let yielded_events = trigger.evaluate(posted).await?;
- if yielded_events.is_empty() {
- tracing::debug!(message_id = %posted.message_id, "agent trigger yielded no event");
+ Ok(())
}
- for yielded in yielded_events {
- let json = serde_json::to_string(yielded.event())?;
- tracing::info!(yielded_event = %json, "agent trigger yielded event");
- let publish = publisher.send_event(&yielded)?;
- publish.await.context("agent event publication task failed")??;
+ .instrument(span.clone())
+ .await;
+ if let Err(error) = &result {
+ record_span_error(&span, error);
+ }
+ if let Err(error) = result {
+ run_error = Some(error);
+ break;
}
-
- commit_message(&consumer, kafka_message)?;
}
}
}
- Ok(())
+ match run_error {
+ Some(error) => Err(error),
+ None => Ok(()),
+ }
}
diff --git a/services/connection_gateway/Cargo.toml b/services/connection_gateway/Cargo.toml
index 955323dc9c..117b0c1839 100644
--- a/services/connection_gateway/Cargo.toml
+++ b/services/connection_gateway/Cargo.toml
@@ -57,10 +57,12 @@ macro_config = { path = "../../crates/macro_config" }
macro_entrypoint = { path = "../../crates/macro_entrypoint" }
macro_env = { path = "../../crates/macro_env" }
macro_env_var = { path = "../../crates/macro_env_var" }
+macro_tower_layers = { path = "../../crates/macro_tower_layers" }
mockall.workspace = true
model = { path = "../../crates/model" }
model-entity = { path = "../../crates/model-entity" }
models_bulk_upload = { path = "../../crates/models_bulk_upload" }
+opentelemetry = { workspace = true, features = ["trace"] }
redis = { workspace = true, features = [
"aio",
"tokio-comp",
@@ -78,6 +80,7 @@ tokio = { workspace = true }
tower = { workspace = true }
tower-http = { workspace = true }
tracing = { workspace = true, features = ["attributes"] }
+tracing-opentelemetry = { workspace = true }
tungstenite = { workspace = true }
url = { workspace = true }
utoipa = { workspace = true, features = ["axum_extras"] }
diff --git a/services/connection_gateway/src/api/connection.rs b/services/connection_gateway/src/api/connection.rs
index 8441f2b931..797a4b2b29 100644
--- a/services/connection_gateway/src/api/connection.rs
+++ b/services/connection_gateway/src/api/connection.rs
@@ -2,7 +2,9 @@ use crate::{
config::Config,
context::{ApiContext, AppState, AuthorizationService},
model::{
- connection::ConnectionContext, message::OutgoingMessage, tracking::EntityConnectionExt,
+ connection::ConnectionContext,
+ message::{OutgoingMessage, record_span_error},
+ tracking::EntityConnectionExt,
},
};
use anyhow::Result;
@@ -27,6 +29,8 @@ use model::user::UserContext;
use model_entity::EntityType;
use std::sync::Arc;
use tokio::sync::mpsc::Receiver;
+use tracing::Instrument as _;
+use tracing_opentelemetry::OpenTelemetrySpanExt as _;
mod messages;
@@ -136,15 +140,37 @@ pub async fn forwarder(
mut sink: SplitSink<WebSocket, AxumWebsocketMessage>,
mut receiver: Receiver<OutgoingMessage>,
) -> Result<()> {
- while let Some(message) = receiver.recv().await {
- if let Ok(msg) = message.try_into()
- && let Err(err) = sink.send(msg).await
- {
- tracing::warn!(
- error=?err,
- "Failed to send message to WebSocket, client likely disconnected"
- );
- break;
+ while let Some(mut message) = receiver.recv().await {
+ let span = match &message {
+ OutgoingMessage::Message(message) => {
+ let span = tracing::info_span!(
+ "connection_gateway.websocket_send",
+ otel.kind = "producer",
+ message_type = %message.message_type,
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ if let Some(parent) = message.remote_trace_context() {
+ let _ = span.set_parent(parent);
+ }
+ span
+ }
+ OutgoingMessage::Pong => tracing::Span::none(),
+ };
+ if let OutgoingMessage::Message(message) = &mut message {
+ message.trace.clear();
+ }
+
+ if let Ok(msg) = message.try_into() {
+ let result = sink.send(msg).instrument(span.clone()).await;
+ if let Err(err) = result {
+ record_span_error(&span, &err);
+ tracing::warn!(
+ error=?err,
+ "Failed to send message to WebSocket, client likely disconnected"
+ );
+ break;
+ }
}
}
diff --git a/services/connection_gateway/src/api/message.rs b/services/connection_gateway/src/api/message.rs
index 9763352c65..bc6f0c4828 100644
--- a/services/connection_gateway/src/api/message.rs
+++ b/services/connection_gateway/src/api/message.rs
@@ -61,10 +61,7 @@ pub async fn send_message_handler(
let res = send_message_to_entity(
&ctx,
&entity,
- Message {
- message_type: body.message_type.clone(),
- data: body.message.to_string(),
- },
+ Message::new(body.message_type.clone(), body.message.to_string()),
redis_connection,
)
.await
@@ -110,10 +107,7 @@ pub async fn batch_send_message_handler(
send_message_to_entity(
&ctx,
entity,
- Message {
- message_type: body.message_type.clone(),
- data: body.message.to_string(),
- },
+ Message::new(body.message_type.clone(), body.message.to_string()),
redis_connection,
)
}))
@@ -165,10 +159,10 @@ pub async fn batch_send_unique_messages_handler(
send_message_to_entity(
&ctx,
&message.entity,
- Message {
- message_type: message.message_type.clone(),
- data: message.message_content.to_string(),
- },
+ Message::new(
+ message.message_type.clone(),
+ message.message_content.to_string(),
+ ),
redis_connection,
)
}))
diff --git a/services/connection_gateway/src/main.rs b/services/connection_gateway/src/main.rs
index 6a35139c33..736b418520 100644
--- a/services/connection_gateway/src/main.rs
+++ b/services/connection_gateway/src/main.rs
@@ -35,6 +35,7 @@ use macro_auth::middleware::decode_jwt::JwtValidationArgs;
use macro_authorization::{InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState};
use macro_entrypoint::MacroEntrypoint;
use macro_env::Environment;
+use macro_tower_layers::MacroRequestIdAndTracingLayer;
use service::dynamodb::create_dynamo_db_connection_manager;
use service::redis::poll_messages;
use sqlx::postgres::PgPoolOptions;
@@ -140,6 +141,7 @@ async fn main() -> Result<()> {
Duration::from_secs(60),
)),
})
+ .layer(MacroRequestIdAndTracingLayer::new(Duration::from_millis(200)).into_inner())
.layer(cors);
tracing::info!(
diff --git a/services/connection_gateway/src/model/message.rs b/services/connection_gateway/src/model/message.rs
index 588bf213be..382152e5f7 100644
--- a/services/connection_gateway/src/model/message.rs
+++ b/services/connection_gateway/src/model/message.rs
@@ -1,12 +1,49 @@
use anyhow::Result;
+use opentelemetry::trace::TraceContextExt as _;
use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
use stream::domain::StreamEvent;
+use tracing_opentelemetry::OpenTelemetrySpanExt as _;
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Message {
#[serde(rename = "type")]
pub message_type: String,
pub data: String,
+ #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+ pub(crate) trace: HashMap<String, String>,
+}
+
+impl Message {
+ pub fn new(message_type: String, data: String) -> Self {
+ Self {
+ message_type,
+ data,
+ trace: HashMap::new(),
+ }
+ .with_current_trace_context()
+ }
+
+ pub fn with_current_trace_context(mut self) -> Self {
+ self.trace.clear();
+ let context = tracing::Span::current().context();
+ opentelemetry::global::get_text_map_propagator(|propagator| {
+ propagator.inject_context(&context, &mut self.trace);
+ });
+ self
+ }
+
+ pub fn remote_trace_context(&self) -> Option<opentelemetry::Context> {
+ let context = opentelemetry::global::get_text_map_propagator(|propagator| {
+ propagator.extract(&self.trace)
+ });
+ context.span().span_context().is_valid().then_some(context)
+ }
+}
+
+pub(crate) fn record_span_error(span: &tracing::Span, error: &(impl std::fmt::Display + ?Sized)) {
+ span.record("otel.status_code", "ERROR");
+ span.record("otel.status_description", tracing::field::display(error));
}
static STREAM_EVENT_TYPE: &str = "stream_event";
@@ -14,10 +51,7 @@ impl TryFrom<StreamEvent> for Message {
type Error = anyhow::Error;
fn try_from(value: StreamEvent) -> Result<Self, Self::Error> {
serde_json::to_string(&value)
- .map(|data| Self {
- data,
- message_type: STREAM_EVENT_TYPE.into(),
- })
+ .map(|data| Self::new(STREAM_EVENT_TYPE.into(), data))
.map_err(anyhow::Error::from)
}
}
diff --git a/services/connection_gateway/src/model/stream.rs b/services/connection_gateway/src/model/stream.rs
index 8122247b7c..05d06af73e 100644
--- a/services/connection_gateway/src/model/stream.rs
+++ b/services/connection_gateway/src/model/stream.rs
@@ -6,11 +6,7 @@ const STREAM_MESSAGE_TYPE: &str = "stream";
impl TryFrom<StreamItem> for OutgoingMessage {
type Error = serde_json::Error;
fn try_from(value: StreamItem) -> Result<Self, Self::Error> {
- serde_json::to_string(&value).map(|data| {
- OutgoingMessage::Message(Message {
- message_type: STREAM_MESSAGE_TYPE.into(),
- data,
- })
- })
+ serde_json::to_string(&value)
+ .map(|data| OutgoingMessage::Message(Message::new(STREAM_MESSAGE_TYPE.into(), data)))
}
}
diff --git a/services/connection_gateway/src/service/redis.rs b/services/connection_gateway/src/service/redis.rs
index ba257f5ed2..5ab142e226 100644
--- a/services/connection_gateway/src/service/redis.rs
+++ b/services/connection_gateway/src/service/redis.rs
@@ -1,8 +1,13 @@
use anyhow::{Context, Result};
use futures::StreamExt;
use redis::{AsyncCommands, FromRedisValue, ParsingError, Value, aio::MultiplexedConnection};
+use tracing::Instrument as _;
+use tracing_opentelemetry::OpenTelemetrySpanExt as _;
-use crate::{context::ApiContext, model::message::Message};
+use crate::{
+ context::ApiContext,
+ model::message::{Message, record_span_error},
+};
pub const REDIS_CHANNEL: &str = "connection_gateway.messages";
@@ -19,16 +24,31 @@ pub struct MessageWithConnection {
/// will handle sending the message to the client correctly.
pub async fn post_message(
mut connection: MultiplexedConnection,
- message: MessageWithConnection,
+ mut message: MessageWithConnection,
) -> Result<()> {
- let message_json = serde_json::to_string(&message).context("Failed to serialize message")?;
-
- connection
- .publish::<&str, &str, ()>(REDIS_CHANNEL, message_json.as_str())
- .await
- .context("Failed to publish message")?;
-
- Ok(())
+ let span = tracing::info_span!(
+ "connection_gateway.redis_publish",
+ otel.kind = "producer",
+ message_type = %message.message.message_type,
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ let result = async {
+ message.message = message.message.with_current_trace_context();
+ let message_json =
+ serde_json::to_string(&message).context("Failed to serialize message")?;
+
+ connection
+ .publish::<&str, &str, ()>(REDIS_CHANNEL, message_json.as_str())
+ .await
+ .context("Failed to publish message")
+ }
+ .instrument(span.clone())
+ .await;
+ if let Err(error) = &result {
+ record_span_error(&span, error);
+ }
+ result
}
impl FromRedisValue for MessageWithConnection {
@@ -55,7 +75,7 @@ pub async fn poll_messages(ctx: ApiContext) -> Result<()> {
.context("Failed to subscribe to reddis channel")?;
while let Some(maybe_message) = stream.next().await {
- let message: MessageWithConnection =
+ let mut message: MessageWithConnection =
match maybe_message.get_payload::<MessageWithConnection>() {
Ok(msg) => msg,
Err(err) => {
@@ -79,12 +99,26 @@ pub async fn poll_messages(ctx: ApiContext) -> Result<()> {
connection_id = message.connection_id,
"received message from redis, sending to connection"
);
-
- if let Err(err) = ctx
- .connection_manager
- .send_message(message.connection_id.as_str(), message.message)
- .await
- {
+ let span = tracing::info_span!(
+ "connection_gateway.redis_dispatch",
+ otel.kind = "consumer",
+ message_type = %message.message.message_type,
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ if let Some(parent) = message.message.remote_trace_context() {
+ let _ = span.set_parent(parent);
+ }
+ let result = async {
+ message.message = message.message.with_current_trace_context();
+ ctx.connection_manager
+ .send_message(message.connection_id.as_str(), message.message)
+ .await
+ }
+ .instrument(span.clone())
+ .await;
+ if let Err(err) = result {
+ record_span_error(&span, &err);
tracing::error!(error=?err, "failed to send message");
}
}
diff --git a/services/connection_gateway/src/service/tracker.rs b/services/connection_gateway/src/service/tracker.rs
index f7a6451f1d..2e6fdcbfd0 100644
--- a/services/connection_gateway/src/service/tracker.rs
+++ b/services/connection_gateway/src/service/tracker.rs
@@ -150,14 +150,14 @@ pub struct UserTrackingChange {
pub async fn notify_tracking_change(ctx: ConnectionContext<'_>, entity: &Entity<'_>) -> Result<()> {
let users = get_users_for_entity(ctx, entity, Some(DEFAULT_TIMEOUT_THRESHOLD)).await?;
- let message: Message = Message {
- message_type: "user_tracking_change".to_string(),
- data: serde_json::to_string(&UserTrackingChange {
+ let message = Message::new(
+ "user_tracking_change".to_string(),
+ serde_json::to_string(&UserTrackingChange {
entity_type: entity.entity_type,
entity_id: entity.entity_id.to_string(),
user_ids: users,
})?,
- };
+ );
let redis_connection = ctx.api_context.redis_connection.clone();
send_message_to_entity(ctx, entity, message, redis_connection).await?;
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index afc418e199..3422c41102 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -167,7 +167,13 @@ maybe_env_vars! {
#[tokio::main]
async fn main() -> anyhow::Result<()> {
- MacroEntrypoint::default().init();
+ let entrypoint = MacroEntrypoint::default().init();
+ let result = run().await;
+ entrypoint.shutdown();
+ result
+}
+
+async fn run() -> anyhow::Result<()> {
let env = Environment::new_or_prod();
let aws_config = macro_aws_config::get_macro_aws_config().await;