Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(agent-harness): route commands through Redis (#6195)
Wolf Mermelstein48a31c227652
diff --git a/Cargo.lock b/Cargo.lockindex d2e4f6fe51..dfe02fedda 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -264,7 +264,9 @@ dependencies = ["macro_uuid","mockall","model_user",+ "opentelemetry","pipedream_mcp",+ "redis","reqwest 0.13.4","rootcause","rustls 0.23.40",@@ -276,6 +278,7 @@ dependencies = ["tokio-tungstenite 0.28.0","tokio-util","tracing",+ "tracing-opentelemetry","url","workspace-hack",]@@ -307,6 +310,7 @@ dependencies = ["cursor_cloud_agents","database_env_vars","entity_access",+ "futures","github","harness_id","kafka_util",@@ -328,13 +332,14 @@ dependencies = ["notification","pipedream_mcp","rdkafka",- "reqwest 0.13.4",+ "redis","rootcause","secretsmanager_client","serde","serde_json","sqlx","tokio",+ "tokio-retry","tokio-util","tower 0.5.3","tracing",diff --git a/crates/agent_harness/Cargo.toml b/crates/agent_harness/Cargo.tomlindex f1f07cd120..351a6071df 100644--- a/crates/agent_harness/Cargo.toml+++ b/crates/agent_harness/Cargo.toml@@ -34,7 +34,9 @@ macro_authorization = { path = "../macro_authorization", features = ["axum"] }macro_tower_layers = { path = "../macro_tower_layers" }macro_user_id = { path = "../macro_user_id" }macro_uuid = { path = "../macro_uuid" }+opentelemetry = { workspace = true, features = ["trace"] }reqwest = { workspace = true }+redis = { workspace = true, features = ["aio", "tokio-comp"] }rootcause = { workspace = true }rustls = { workspace = true }serde = { workspace = true }@@ -51,6 +53,7 @@ tokio = { workspace = true, features = [tokio-util = { workspace = true, features = ["rt"] }tokio-tungstenite = { workspace = true }tracing = { workspace = true }+tracing-opentelemetry = { workspace = true }url = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/agent_harness/src/domain/error.rs b/crates/agent_harness/src/domain/error.rsindex b255752516..9a2105cfeb 100644--- a/crates/agent_harness/src/domain/error.rs+++ b/crates/agent_harness/src/domain/error.rs@@ -63,11 +63,6 @@ pub enum HarnessError {/// A prompt could not be composed for the agent runtime.#[error("failed to compose agent prompt: {0}")]PromptComposition(rootcause::Report),- /// A live peer manages the session but published no address to forward- /// to. Transient by construction: only a replica from before addresses- /// existed, or one whose first heartbeat has not landed, has no address.- #[error("agent session {0} is managed by a live replica with no forwarding address")]- ManagerUnreachable(AgentSessionId),/// Forwarding a command to the session's managing replica failed.#[error("failed to forward an agent session command: {0}")]Forward(rootcause::Report),diff --git a/crates/agent_harness/src/domain/ports.rs b/crates/agent_harness/src/domain/ports.rsindex 0f8416259b..916d636145 100644--- a/crates/agent_harness/src/domain/ports.rs+++ b/crates/agent_harness/src/domain/ports.rs@@ -5,7 +5,7 @@ use std::pin::Pin;use std::sync::Arc;use agent_session::domain::connection::RuntimeAttachment;-use agent_session::domain::model::{AgentSessionId, ReplicaAddress, SandboxSize};+use agent_session::domain::model::{AgentSessionId, SandboxSize};use agent_session::domain::ports::AgentConnector;use bot_id::BotId;use harness_id::HarnessId;@@ -19,22 +19,23 @@ use super::model::{};use super::sandbox::SandboxResizeEffect;-/// Delivers a session's command to the replica that manages its live actor.-///-/// The receiving side executes without re-resolving management (a forward is-/// single-hop by contract, so two replicas with momentarily different views-/// cannot bounce a command between each other). Success means the peer ran-/// the command to completion - the response is the acknowledgment - so a-/// caller that awaited a forward has the same guarantee as one that executed-/// locally.+/// The distributed destination for a forwarded command.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum CommandTarget {+ /// The replica that owns the session actor.+ Replica(agent_session::domain::model::ReplicaId),+ /// The replica holding a registered harness's runtime socket.+ Harness(HarnessId),+}++/// Forwards commands to the replica currently responsible for execution.pub trait CommandForwarder: Send + Sync + 'static {- /// Run `command` for `session` on the replica at `target`, reporting- /// what that replica's execution did with it.+ /// Run `command` at `target`.fn forward(&self,- target: &ReplicaAddress,session: AgentSessionId,command: HarnessCommand,+ target: CommandTarget,) -> impl Future<Output = Result<CommandOutcome>> + Send;}@@ -47,13 +48,11 @@ pub struct NoPeers;impl CommandForwarder for NoPeers {async fn forward(&self,- target: &ReplicaAddress,session: AgentSessionId,_command: HarnessCommand,+ _target: CommandTarget,) -> Result<CommandOutcome> {- Err(HarnessError::Forward(rootcause::report!(- "this deployment has no command forwarding, yet {target} manages session {session}"- )))+ Err(HarnessError::Disconnected(session))}}@@ -170,6 +169,9 @@ pub trait RuntimeConnections: Send + Sync + 'static {&self,bot: BotId,) -> impl Future<Output = anyhow::Result<Option<HarnessId>>> + Send;++ /// Whether this process currently holds `harness`'s physical runtime socket.+ fn is_connected(&self, harness: HarnessId) -> bool;}/// Mints the one secret a sandbox is given, and the config that points it atdiff --git a/crates/agent_harness/src/domain/service.rs b/crates/agent_harness/src/domain/service.rsindex 6a5eb9bf45..b6ff47a4ee 100644--- a/crates/agent_harness/src/domain/service.rs+++ b/crates/agent_harness/src/domain/service.rs@@ -59,20 +59,20 @@ struct QueuedCommand {trait ErasedForwarder: Send + Sync + 'static {fn forward<'a>(&'a self,- target: &'a agent_session::domain::model::ReplicaAddress,session: AgentSessionId,command: HarnessCommand,+ target: crate::domain::ports::CommandTarget,) -> std::pin::Pin<Box<dyn Future<Output = Result<CommandOutcome>> + Send + 'a>>;}impl<F: CommandForwarder> ErasedForwarder for F {fn forward<'a>(&'a self,- target: &'a agent_session::domain::model::ReplicaAddress,session: AgentSessionId,command: HarnessCommand,+ target: crate::domain::ports::CommandTarget,) -> std::pin::Pin<Box<dyn Future<Output = Result<CommandOutcome>> + Send + 'a>> {- Box::pin(CommandForwarder::forward(self, target, session, command))+ Box::pin(CommandForwarder::forward(self, session, command, target))}}@@ -337,10 +337,10 @@ where}}-/// The receiving half of command forwarding: what the internal forward route-/// calls, implemented by the harness as [`AgentHarnessService::execute_here`].+/// The receiving half of command forwarding, called by the command-bus+/// consumer and implemented by the harness as [`AgentHarnessService::execute_here`].pub trait ForwardedCommands: Send + Sync + 'static {- /// Run a command a peer already routed to this replica.+ /// Run a command the transport has targeted to this replica.fn execute_forwarded(&self,session_id: AgentSessionId,@@ -761,6 +761,7 @@ fn queue_result<T>(fn into_session_error(error: HarnessError) -> AgentSessionError {match error {HarnessError::Session(error) => error,+ HarnessError::Disconnected(session) => AgentSessionError::Disconnected(session),other => AgentSessionError::Unknown(anyhow::anyhow!(other)),}}@@ -786,12 +787,6 @@ where{/// Execute where the session's live actor is: locally when nobody (or/// this replica) manages it, on the managing peer otherwise.- ///- /// A failed forward re-reads the lease once: the one legitimate reason a- /// live manager refuses its own session is that it died mid-flight, and- /// then its heartbeat going stale is what lets this replica take over. A- /// manager that is alive but unreachable stays an error - executing- /// locally anyway is how two actors end up on one session./// The routing decision is recorded on the span, not only logged: which of/// the three answers the lease gave, which peer it named, and whether the/// command left this process. Those are the fields you group by when a@@ -804,7 +799,6 @@ whereagent.session.management = tracing::field::Empty,agent.session.manager_replica = tracing::field::Empty,agent.command.forwarded = tracing::field::Empty,- agent.command.stale_fallback = tracing::field::Empty,))]async fn route_then_execute(@@ -824,7 +818,30 @@ whereSessionManagement::Unmanaged => {span.record("agent.session.management", "unmanaged");span.record("agent.command.forwarded", false);- return self.execute(session_id, command).await;+ let session = self.sessions.get_session(session_id).await?;+ if AgentKind::for_session(session.bot_id, &session.harness) != AgentKind::External {+ return self.execute(session_id, command).await;+ }+ let Some(harness) = self+ .runtimes+ .bound_harness(session.bot_id)+ .await+ .map_err(AgentSessionError::Unknown)?+ else {+ return self.execute(session_id, command).await;+ };+ if self.runtimes.is_connected(harness) {+ return self.execute(session_id, command).await;+ }+ span.record("agent.command.forwarded", true);+ return self+ .forwarder+ .forward(+ session_id,+ command,+ crate::domain::ports::CommandTarget::Harness(harness),+ )+ .await;}SessionManagement::Ours => {span.record("agent.session.management", "ours");@@ -838,36 +855,15 @@ where"agent.session.manager_replica",tracing::field::display(manager.replica),);- let Some(address) = manager.address else {- // Recorded false deliberately: the command stayed here, but as an- // error rather than a local execution.- span.record("agent.command.forwarded", false);- return Err(HarnessError::ManagerUnreachable(session_id));- };span.record("agent.command.forwarded", true);tracing::info!(%session_id, peer = %manager.replica, "forwarding an agent session command");- match self- .forwarder- .forward(&address, session_id, command.clone())+ self.forwarder+ .forward(+ session_id,+ command,+ crate::domain::ports::CommandTarget::Replica(manager.replica),+ ).await- {- Ok(outcome) => Ok(outcome),- Err(forward_error) => match self.sessions.management(session_id).await? {- SessionManagement::Unmanaged | SessionManagement::Ours => {- // Worth aggregating rather than only logging: routine- // fallbacks mean heartbeats are not keeping up, which is a- // different problem from an occasional dead peer.- span.record("agent.command.stale_fallback", true);- tracing::warn!(- error = ?forward_error,- %session_id,- "the managing replica went stale mid-forward; executing locally"- );- self.execute(session_id, command).await- }- SessionManagement::Peer(_) => Err(forward_error),- },- }}async fn execute(diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rsindex 90ef3edf5c..df2a33d888 100644--- a/crates/agent_harness/src/domain/service/test.rs+++ b/crates/agent_harness/src/domain/service/test.rs@@ -31,6 +31,7 @@ use macro_uuid::Uuid;use tokio::sync::mpsc;use super::AgentHarnessService;+use super::into_session_error;use crate::domain::error::HarnessError;use crate::domain::model::{AgentKind, AgentRuntimeConfig, AnnounceOrigin, CommandOutcome, DeliverAction, HarnessCommand,@@ -54,6 +55,14 @@ fn sender() -> MacroUserIdStr<'static> {MacroUserIdStr::try_from_email("[email protected]").expect("a valid user id")}+#[test]+fn disconnected_harness_errors_keep_the_session_error_classification() {+ assert!(matches!(+ into_session_error(HarnessError::Disconnected(AgentSessionId::TEST_A)),+ AgentSessionError::Disconnected(AgentSessionId::TEST_A)+ ));+}+/// A sender whose email domain is `is_macro_staff` - the identity the/// Daytona staff gate admits.fn staff_sender() -> MacroUserIdStr<'static> {@@ -1856,7 +1865,7 @@ async fn a_disconnected_external_session_never_gets_a_sandbox() {assert!(matches!(error,- HarnessError::Session(AgentSessionError::Disconnected(id)) if id == session.id,+ HarnessError::Disconnected(id) if id == session.id,),"got {error:?}");@@ -2147,78 +2156,51 @@ async fn set_sandbox_size_unsupported_does_not_persist() {assert_eq!(containers.resumed(), 0);}+type ForwardCall = (crate::domain::ports::CommandTarget, AgentSessionId);+/// A [`CommandForwarder`] that records its calls and reports success.#[derive(Clone, Default)]struct RecordingForwarder {- calls: Arc<Mutex<Vec<(String, AgentSessionId)>>>,+ calls: Arc<Mutex<Vec<ForwardCall>>>,}impl crate::domain::ports::CommandForwarder for RecordingForwarder {async fn forward(&self,- target: &agent_session::domain::model::ReplicaAddress,session: AgentSessionId,_command: HarnessCommand,+ target: crate::domain::ports::CommandTarget,) -> crate::domain::error::Result<CommandOutcome> {- self.calls- .lock()- .unwrap()- .push((target.as_str().to_owned(), session));+ self.calls.lock().unwrap().push((target, session));Ok(CommandOutcome::Completed)}}-/// A forwarder standing in for a peer that died mid-forward: the call fails,-/// and by the time the caller re-reads the lease the peer's claim is gone.-#[derive(Clone)]-struct DyingPeerForwarder {- repo: InMemoryAgentSessionRepo,- claim: agent_session::domain::model::SessionClaim,-}--impl crate::domain::ports::CommandForwarder for DyingPeerForwarder {- async fn forward(- &self,- _target: &agent_session::domain::model::ReplicaAddress,- _session: AgentSessionId,- _command: HarnessCommand,- ) -> crate::domain::error::Result<CommandOutcome> {- use agent_session::domain::ports::SessionOwnership as _;- self.repo.release(&self.claim).await.expect("peer releases");- Err(HarnessError::Forward(rootcause::report!(- "the peer went away"- )))- }-}--/// Claim a session for a fabricated peer replica that publishes an address.+/// Claim a session for a fabricated peer replica.async fn claim_as_peer(repo: &InMemoryAgentSessionRepo,session: AgentSessionId,- address: &str,) -> agent_session::domain::model::SessionClaim {- use agent_session::domain::model::{ClaimOutcome, ReplicaAddress, ReplicaId};+ use agent_session::domain::model::{ClaimOutcome, ReplicaId};use agent_session::domain::ports::SessionOwnership as _;let peer = ReplicaId::mint();- repo.heartbeat(peer, Some(&ReplicaAddress::new(address)))- .await- .expect("peer heartbeats");+ repo.heartbeat(peer, None).await.expect("peer heartbeats");match repo.claim(session, peer).await.expect("peer claims") {ClaimOutcome::Claimed(claim) => claim,ClaimOutcome::ManagedElsewhere(_) => panic!("nobody else should hold the test session"),}}-/// A command for a session a live peer manages goes to the peer's address+/// A command for a session a live peer manages goes through the command bus/// and never executes here: the session outlives a Delete this replica would/// otherwise have applied.#[tokio::test]-async fn commands_for_a_peer_managed_session_forward_to_its_address() {+async fn commands_for_a_peer_managed_session_forward_through_redis() {let repo = InMemoryAgentSessionRepo::new();let session = agent_session::testing::test_agent_session(AgentSessionId::new());let id = session.id;repo.insert_session(session);- let _claim = claim_as_peer(&repo, id, "http://10.0.0.7:8100").await;+ let claim = claim_as_peer(&repo, id).await;let forwarder = RecordingForwarder::default();let service = AgentHarnessService::new(AgentSessionServiceImpl::new(@@ -2246,26 +2228,23 @@ async fn commands_for_a_peer_managed_session_forward_to_its_address() {.await.expect("the forwarded command succeeds");- assert_eq!(- forwarder.calls.lock().unwrap().clone(),- vec![("http://10.0.0.7:8100".to_owned(), id)]- );+ assert_eq!(forwarder.calls.lock().unwrap().len(), 1);+ assert!(matches!(+ forwarder.calls.lock().unwrap()[0],+ (crate::domain::ports::CommandTarget::Replica(replica), session)+ if replica == claim.replica && session == id+ ));assert!(repo.get(id).await.is_ok(),"the delete ran on the peer, not here");}-/// A forward that fails against a peer whose lease is gone by the re-read-/// falls back to executing locally, so a peer crash costs one retry rather-/// than the command.#[tokio::test]-async fn a_dead_peers_command_falls_back_to_local_execution() {+async fn unmanaged_external_session_forwards_to_its_remote_harness() {let repo = InMemoryAgentSessionRepo::new();- let session = agent_session::testing::test_agent_session(AgentSessionId::new());- let id = session.id;- repo.insert_session(session);- let claim = claim_as_peer(&repo, id, "http://10.0.0.7:8100").await;+ let runtimes = TestConnections::new(MirrorBindings, RuntimeRegistry::new());+ let forwarder = RecordingForwarder::default();let service = AgentHarnessService::new(AgentSessionServiceImpl::new(repo.clone(),@@ -2274,14 +2253,11 @@ async fn a_dead_peers_command_falls_back_to_local_execution() {),MockContainerManager::new(),AnnouncerMock::new(),- TestConnections::new(MirrorBindings, RuntimeRegistry::<ContainerSender>::new()),+ runtimes,PromptContextMock::default(),PromptComposerMock::default(),EgressProvisionerMock::new(),- DyingPeerForwarder {- repo: repo.clone(),- claim,- },+ forwarder.clone(),SessionDefaults {bot_id: BotId::TEST_A,model: "claude".to_owned(),@@ -2289,14 +2265,25 @@ async fn a_dead_peers_command_falls_back_to_local_execution() {repo_url: "https://github.com/macro-inc/macro".to_owned(),},);+ let session = service+ .open_external_session(open_external_request("/srv/agent"))+ .await+ .expect("external session opens without a local runtime");service- .execute(id, HarnessCommand::Delete)+ .execute(session.id, HarnessCommand::Delete).await- .expect("the fallback executes locally");+ .expect("the remote harness executes the command");+ assert_eq!(+ *forwarder.calls.lock().unwrap(),+ [(+ crate::domain::ports::CommandTarget::Harness(harness_for_bot(session.bot_id)),+ session.id,+ )]+ );assert!(- repo.get(id).await.is_err(),- "the delete ran here once the peer was gone"+ repo.get(session.id).await.is_ok(),+ "the command did not execute on this replica");}diff --git a/crates/agent_harness/src/inbound/forward.rs b/crates/agent_harness/src/inbound/forward.rsdeleted file mode 100644index 6093e9523a..0000000000--- a/crates/agent_harness/src/inbound/forward.rs+++ /dev/null@@ -1,127 +0,0 @@-//! The receiving half of replica-to-replica command forwarding.-//!-//! A peer that consumed a command for a session this replica manages POSTs it-//! here. Internal-only by construction: the caller is another replica of this-//! same deployment, authenticated with the shared internal key, never a user.-//! The handler executes without re-resolving management - forwarding is-//! single-hop, and this endpoint is the second hop honoring that contract.--use std::sync::Arc;--use agent_session::domain::model::AgentSessionId;-use axum::Router;-use axum::extract::{FromRef, Json, Path, State};-use axum::http::StatusCode;-use axum::response::{IntoResponse, Response};-use axum::routing::post;-use macro_authorization::{- InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService, MacroAuthorizationState,-};-use macro_uuid::Uuid;--use crate::domain::error::HarnessError;-use crate::domain::model::HarnessCommand;-use crate::domain::service::ForwardedCommands;--/// State for the command-forwarding route.-pub struct ForwardGatewayState<Harness, Auth> {- harness: Arc<Harness>,- authorization_state: MacroAuthorizationState<Auth>,-}--impl<Harness, Auth> ForwardGatewayState<Harness, Auth> {- /// Create forwarding state.- pub fn new(harness: Arc<Harness>, authorization_state: MacroAuthorizationState<Auth>) -> Self {- Self {- harness,- authorization_state,- }- }-}--// Manual Clone impl: everything is behind an Arc.-impl<Harness, Auth> Clone for ForwardGatewayState<Harness, Auth> {- fn clone(&self) -> Self {- Self {- harness: Arc::clone(&self.harness),- authorization_state: self.authorization_state.clone(),- }- }-}--impl<Harness, Auth> FromRef<ForwardGatewayState<Harness, Auth>> for MacroAuthorizationState<Auth> {- fn from_ref(state: &ForwardGatewayState<Harness, Auth>) -> Self {- state.authorization_state.clone()- }-}--/// Build the router serving `POST /agent-sessions/{id}/command`. Mount it-/// under `/internal`.-pub fn forward_router<Harness, Auth, S>(state: ForwardGatewayState<Harness, Auth>) -> Router<S>-where- Harness: ForwardedCommands,- Auth: MacroAuthorizationService,- S: Clone + Send + Sync + 'static,-{- Router::new()- .route(- "/agent-sessions/{session_id}/command",- post(forward_handler::<Harness, Auth>),- )- .with_state(state)-}--/// Execute a command a peer replica forwarded to this one.-///-/// Instrumented because this is the far half of the hop: the request carries-/// the sending replica's `traceparent`, which the tracing layer turns into a-/// parent link, so this span shows a forwarded command executing under the-/// span that routed it instead of as an unrelated root. `outcome` distinguishes-/// the three answers the sender can get without reading the response body.-#[tracing::instrument(- name = "harness.forward.receive",- skip_all,- fields(- agent.session.id = %session_id,- agent.command.forwarded = true,- agent.command.outcome = tracing::field::Empty,- )-)]-async fn forward_handler<Harness, Auth>(- State(state): State<ForwardGatewayState<Harness, Auth>>,- _caller: MacroAuthorizationExtractor<Auth, InternalOnly>,- Path(session_id): Path<Uuid>,- Json(command): Json<HarnessCommand>,-) -> Response-where- Harness: ForwardedCommands,- Auth: MacroAuthorizationService,-{- let session_id = AgentSessionId::new_from_uuid(session_id);- let span = tracing::Span::current();- match state.harness.execute_forwarded(session_id, command).await {- // The outcome rides back in the body so the sender can answer its own- // caller honestly - a queued deliver is not a delivered one.- Ok(outcome) => (StatusCode::OK, axum::Json(outcome)).into_response(),- // Surfaced with its own status so the sender can tell "the actor is- // not here after all" from an execution failure: the sender's- // fallback re-reads the lease on any error, but a 409 is the signal- // that its view was stale rather than that the command is bad.- Err(HarnessError::Disconnected(id))- | Err(HarnessError::Session(- agent_session::domain::error::AgentSessionError::Disconnected(id),- )) => {- span.record("agent.command.outcome", "not_attached_here");- (- StatusCode::CONFLICT,- format!("session {id} is not attached here"),- )- .into_response()- }- Err(error) => {- span.record("agent.command.outcome", "failed");- tracing::error!(error = ?error, %session_id, "a forwarded command failed");- (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response()- }- }-}diff --git a/crates/agent_harness/src/inbound/mod.rs b/crates/agent_harness/src/inbound/mod.rsindex aeac45aac8..856caaa13c 100644--- a/crates/agent_harness/src/inbound/mod.rs+++ b/crates/agent_harness/src/inbound/mod.rs@@ -3,6 +3,5 @@//! Thin by construction - decode the broker message, ask the domain whether it//! is addressed to us, hand one value inward. No policy lives here.-pub mod forward;pub mod kafka;pub mod runtime_gateway;diff --git a/crates/agent_harness/src/outbound/forward.rs b/crates/agent_harness/src/outbound/forward.rsindex a408d9ce5f..9535bfb455 100644--- a/crates/agent_harness/src/outbound/forward.rs+++ b/crates/agent_harness/src/outbound/forward.rs@@ -1,104 +1,148 @@-//! HTTP implementation of [`CommandForwarder`]: one authenticated POST to the-//! managing replica's internal forward route, awaited for its result.-//!-//! Direct replica-to-replica, not through the load balancer: the target-//! address is the peer's own private URL as published with its heartbeat, so-//! the command lands on exactly the process holding the session's actor.+//! Redis implementation of [`CommandForwarder`].-use agent_session::domain::model::{AgentSessionId, ReplicaAddress};-use reqwest::StatusCode;+use agent_session::domain::model::AgentSessionId;+use harness_id::HarnessId;+use redis::AsyncCommands as _;+use std::collections::HashMap;+use tracing_opentelemetry::OpenTelemetrySpanExt as _;use crate::domain::error::{HarnessError, Result};use crate::domain::model::{CommandOutcome, HarnessCommand};-use crate::domain::ports::CommandForwarder;+use crate::domain::ports::{CommandForwarder, CommandTarget};-/// Header carrying the deployment's shared internal key, as-/// `macro_authorization`'s internal extractor reads it.-const INTERNAL_API_KEY_HEADER: &str = macro_authorization::INTERNAL_API_KEY_HEADER;--/// Forwards commands to peer replicas over their internal HTTP surface.+/// Broadcasts commands to the responsible harness replica over Redis.#[derive(Clone)]-pub struct HttpCommandForwarder {- client: reqwest::Client,- internal_api_key: String,+pub struct RedisCommandForwarder {+ redis: redis::Client,}-impl HttpCommandForwarder {- /// A forwarder authenticating with the deployment's internal key.- ///- /// The timeout bounds the whole forwarded execution, not just the dial:- /// the peer replies only once the command ran, and a Deliver can sit- /// behind a sandbox resume, so this is generous rather than snappy.- pub fn new(internal_api_key: String) -> Result<Self> {- let client = reqwest::Client::builder()- .connect_timeout(std::time::Duration::from_secs(2))- .timeout(std::time::Duration::from_secs(120))- .build()- .map_err(|error| HarnessError::Forward(rootcause::report!(error).into_dynamic()))?;- Ok(Self {- client,- internal_api_key,- })+impl RedisCommandForwarder {+ /// Build a Redis command forwarder.+ pub fn new(redis: redis::Client) -> Self {+ Self { redis }}}-impl CommandForwarder for HttpCommandForwarder {- #[tracing::instrument(- err,- skip(self, command),- fields(- %target,- %session,- http.response.status_code = tracing::field::Empty,- )- )]+impl CommandForwarder for RedisCommandForwarder {async fn forward(&self,- target: &ReplicaAddress,session: AgentSessionId,command: HarnessCommand,+ target: CommandTarget,) -> Result<CommandOutcome> {- let url = format!(- "{}/internal/agent-sessions/{}/command",- target.as_str().trim_end_matches('/'),- session- );- // The peer's inbound layer already extracts `traceparent` and parents- // its request span to the remote span (see `macro_tower_layers`), so- // injecting here is what makes a forwarded command one trace spanning- // both replicas. Without it the hop is two unrelated traces.- let mut headers = reqwest::header::HeaderMap::new();- macro_tower_layers::inject_trace_headers(&mut headers);- let response = self- .client- .post(url)- .header(INTERNAL_API_KEY_HEADER, &self.internal_api_key)- .headers(headers)- .json(&command)- .send()+ let request = RuntimeCommandRequest {+ request_id: macro_uuid::Uuid::new_v4(),+ target: RuntimeCommandTarget::from(target),+ session,+ command,+ trace_context: current_trace_context(),+ };+ let payload = serde_json::to_string(&request)+ .map_err(|error| forward_error("serialize command request", error))?;+ let mut publisher = self+ .redis+ .get_multiplexed_async_connection().await- .map_err(|error| HarnessError::Forward(rootcause::report!(error).into_dynamic()))?;- let status = response.status();- tracing::Span::current().record("http.response.status_code", status.as_u16());- if status.is_success() {- // A body-less success (an older peer's 204, mid-deploy) reads as- // completed: that was the only outcome such a peer could produce.- return Ok(response- .json::<CommandOutcome>()- .await- .unwrap_or(CommandOutcome::Completed));+ .map_err(|error| forward_error("open command publisher", error))?;+ publisher+ .publish::<_, _, ()>(COMMAND_CHANNEL, payload)+ .await+ .map_err(|error| forward_error("publish command request", error))?;+ Ok(CommandOutcome::Completed)+ }+}++/// Redis channel broadcasting runtime commands to every harness replica.+pub const COMMAND_CHANNEL: &str = "agent-harness.runtime-commands";++/// A command broadcast received from another replica.+#[derive(serde::Serialize, serde::Deserialize)]+pub struct RuntimeCommandRequest {+ request_id: macro_uuid::Uuid,+ target: RuntimeCommandTarget,+ session: AgentSessionId,+ command: HarnessCommand,+ #[serde(default, skip_serializing_if = "HashMap::is_empty")]+ trace_context: HashMap<String, String>,+}++/// The process-local destination selected by the command sender.+#[derive(Clone, Copy, serde::Serialize, serde::Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum RuntimeCommandTarget {+ /// A specific session-owning replica.+ Replica(macro_uuid::Uuid),+ /// Whichever replica currently holds this harness runtime connection.+ Harness(HarnessId),+}++impl RuntimeCommandRequest {+ /// The unique ID used to claim and trace this command.+ #[must_use]+ pub fn request_id(&self) -> macro_uuid::Uuid {+ self.request_id+ }++ /// The replica or harness connection selected for execution.+ #[must_use]+ pub fn target(&self) -> RuntimeCommandTarget {+ self.target+ }++ /// The session receiving the command.+ #[must_use]+ pub fn session(&self) -> AgentSessionId {+ self.session+ }++ /// Consume the transport envelope and return its domain command.+ #[must_use]+ pub fn into_command(self) -> HarnessCommand {+ self.command+ }++ /// Create a consumer span parented to the sender's propagated context.+ #[must_use]+ pub fn processing_span(&self) -> tracing::Span {+ runtime_command_span(self)+ }+}++impl From<CommandTarget> for RuntimeCommandTarget {+ fn from(target: CommandTarget) -> Self {+ match target {+ CommandTarget::Replica(replica) => Self::Replica(replica.as_uuid()),+ CommandTarget::Harness(harness) => Self::Harness(harness),}- let body = response.text().await.unwrap_or_default();- // 409 is the peer saying "not attached here after all" - the caller's- // stale-view fallback keys off any error, but keep the distinction- // visible in the report for whoever reads the logs.- let refused = if status == StatusCode::CONFLICT {- "the peer refused the session"- } else {- "the peer failed the command"- };- Err(HarnessError::Forward(rootcause::report!(- "{refused}: {status}: {body}"- )))}}++fn current_trace_context() -> HashMap<String, String> {+ let context = tracing::Span::current().context();+ let mut carrier = HashMap::new();+ opentelemetry::global::get_text_map_propagator(|propagator| {+ propagator.inject_context(&context, &mut carrier);+ });+ carrier+}++fn runtime_command_span(request: &RuntimeCommandRequest) -> tracing::Span {+ let span = tracing::info_span!(+ "redis.process",+ otel.kind = "consumer",+ messaging.system = "redis",+ messaging.operation.name = "process",+ messaging.operation.type = "process",+ messaging.message.id = %request.request_id,+ agent.session.id = %request.session,+ );+ let parent = opentelemetry::global::get_text_map_propagator(|propagator| {+ propagator.extract(&request.trace_context)+ });+ let _ = span.set_parent(parent);+ span+}++fn forward_error(operation: &'static str, error: impl std::fmt::Display) -> HarnessError {+ HarnessError::Forward(rootcause::report!("{operation}: {error}").into_dynamic())+}diff --git a/crates/agent_harness/src/outbound/runtime_registry.rs b/crates/agent_harness/src/outbound/runtime_registry.rsindex 2df9a1e97f..e88bc6e106 100644--- a/crates/agent_harness/src/outbound/runtime_registry.rs+++ b/crates/agent_harness/src/outbound/runtime_registry.rs@@ -183,4 +183,8 @@ whereasync fn bound_harness(&self, bot: BotId) -> anyhow::Result<Option<HarnessId>> {self.bindings.harness_for(bot).await}++ fn is_connected(&self, harness: HarnessId) -> bool {+ self.registry.is_connected(harness)+ }}diff --git a/docker/docker-compose.yml b/docker/docker-compose.ymlindex ca5db7ccb7..6e3094e50b 100644--- a/docker/docker-compose.yml+++ b/docker/docker-compose.yml@@ -351,6 +351,7 @@ services:- /var/run/docker.sock:/var/run/docker.sockdepends_on:- postgres+ - redis- kafka- connection_gateway- lexical_servicediff --git a/infra/stacks/agent-harness-service/Pulumi.dev.yaml b/infra/stacks/agent-harness-service/Pulumi.dev.yamlindex 9188c41f0e..0a203c9756 100644--- a/infra/stacks/agent-harness-service/Pulumi.dev.yaml+++ b/infra/stacks/agent-harness-service/Pulumi.dev.yaml@@ -3,3 +3,4 @@ config:aws:region: us-east-1agent-harness-service:github_sync_app_pem: github-sync-app-pem-devagent-harness-service:macro_api_token_private_secret_key: macro-api-token-private-key-dev+ agent-harness-service:macro_cache_secret_key: macro-cache-devdiff --git a/infra/stacks/agent-harness-service/Pulumi.prod.yaml b/infra/stacks/agent-harness-service/Pulumi.prod.yamlindex d692238d1b..acf0d2e5fa 100644--- a/infra/stacks/agent-harness-service/Pulumi.prod.yaml+++ b/infra/stacks/agent-harness-service/Pulumi.prod.yaml@@ -3,3 +3,4 @@ config:aws:region: us-east-1agent-harness-service:github_sync_app_pem: github-sync-app-pem-prodagent-harness-service:macro_api_token_private_secret_key: macro-api-token-private-key-prod+ agent-harness-service:macro_cache_secret_key: macro-cache-proddiff --git a/infra/stacks/agent-harness-service/agent_harness_service.ts b/infra/stacks/agent-harness-service/agent_harness_service.tsindex b39f2ee6da..2d20f44347 100644--- a/infra/stacks/agent-harness-service/agent_harness_service.ts+++ b/infra/stacks/agent-harness-service/agent_harness_service.ts@@ -64,24 +64,12 @@ type Args = {/*** The agent harness service. Replicated in every environment: each replica- * claims the sessions whose live actors it holds through a fenced Postgres- * lease, and a command landing on the wrong replica is forwarded task-to-task- * over the private network (`POST /internal/agent-sessions/{id}/command`,- * internal-key authenticated) - which is why the service security group- * allows itself ingress on the service port. The Kafka consumer group splits- * partitions across live tasks; the lease plus forwarding is what makes that- * split correct.+ * claims the sessions whose live actors it holds through Postgres ownership,+ * and commands are broadcast through the shared Redis deployment so the+ * responsible replica can execute them. The Kafka consumer group splits+ * partitions across live tasks; ownership plus Redis routing is what makes+ * that split correct.*- * Deploys roll (min healthy 100%, max 200%): the outgoing tasks keep serving- * until their replacements pass health checks, so the control API and egress- * proxy stay up instead of blacking out for the old tasks' sandbox cleanup.- * A deploy therefore doubles the replica count for its duration rather than- * introducing coexistence - replicas coexist all the time, and the lease is- * what makes both the steady state and the overlap correct. A command routed- * to a new task for a session whose live actor is still on an old one- * self-heals through resume. Live sessions already restart across every- * deploy (`shutdown_all` stops each sandbox), so the overlap narrows the- * blast radius rather than widening it.*/export class AgentHarnessService extends pulumi.ComponentResource {public role: aws.iam.Role;@@ -344,9 +332,6 @@ export class AgentHarnessService extends pulumi.ComponentResource {enable: true,rollback: true,},- // Rolling replace: the old task serves until the new one is healthy- // (see the class doc comment for the overlap semantics). A failed- // deploy rolls back with the old task still up rather than at zero.deploymentMinimumHealthyPercent: 100,deploymentMaximumPercent: 200,// Every environment runs two, so dev stays prod-shaped: forwarding is@@ -535,24 +520,6 @@ export class AgentHarnessService extends pulumi.ComponentResource {{ parent: this });- // Replica-to-replica command forwarding: a task that consumed a command- // for a session another replica manages POSTs it directly to that- // replica's private address, bypassing the load balancer.- new aws.vpc.SecurityGroupIngressRule(- `${BASE_NAME}-peer-in`,- {- securityGroupId: serviceSg.id,- description:- 'Allow harness replicas to forward session commands to each other',- referencedSecurityGroupId: serviceSg.id,- fromPort: serviceContainerPort,- toPort: serviceContainerPort,- ipProtocol: 'tcp',- tags: this.tags,- },- { parent: this }- );-new aws.vpc.SecurityGroupEgressRule(`${BASE_NAME}-service-out`,{diff --git a/infra/stacks/agent-harness-service/index.ts b/infra/stacks/agent-harness-service/index.tsindex 205802fe07..0bc17d7cb1 100644--- a/infra/stacks/agent-harness-service/index.ts+++ b/infra/stacks/agent-harness-service/index.ts@@ -45,6 +45,12 @@ const macroApiTokenPrivateKeyArn = aws.secretsmanagerconst MACRO_API_TOKENS = getMacroApiToken();+const MACRO_CACHE = aws.secretsmanager+ .getSecretVersionOutput({+ secretId: config.require('macro_cache_secret_key'),+ })+ .apply((secret) => secret.secretString);+// ── AI tools infra ───────────────────────────────────────────────────────────const aiTools =@@ -109,6 +115,10 @@ const service = new AgentHarnessService(`agent-harness-service-${stack}`, {name: 'ENVIRONMENT',value: stack,},+ {+ name: 'REDIS_URI',+ value: pulumi.interpolate`redis://${MACRO_CACHE}`,+ },// Datadog{name: 'DD_SERVICE',diff --git a/services/agent_harness_service/Cargo.toml b/services/agent_harness_service/Cargo.tomlindex aa92aa17cf..d5b1151f11 100644--- a/services/agent_harness_service/Cargo.toml+++ b/services/agent_harness_service/Cargo.toml@@ -32,6 +32,7 @@ cursor_api_key = { path = "../../crates/cursor_api_key" }cursor_cloud_agents = { path = "../../crates/cursor_cloud_agents" }database_env_vars = { path = "../../crates/database_env_vars" }entity_access = { path = "../../crates/entity_access" }+futures = { workspace = true }github = { path = "../../crates/github", default-features = false, features = ["outbound","ports",@@ -63,8 +64,7 @@ macro_authorization = { path = "../../crates/macro_authorization", default-featu] }macro_cors = { path = "../../crates/macro_cors" }secretsmanager_client = { path = "../../crates/secretsmanager_client" }-reqwest = { workspace = true }-serde_json = { workspace = true }+redis = { workspace = true, features = ["aio", "tokio-comp"] }utoipa = { workspace = true }utoipa-swagger-ui = { workspace = true }macro_config = { path = "../../crates/macro_config" }@@ -73,8 +73,10 @@ macro_env = { path = "../../crates/macro_env" }macro_env_var = { path = "../../crates/macro_env_var" }macro_uuid = { path = "../../crates/macro_uuid" }serde = { workspace = true }+serde_json = { workspace = true }sqlx = { workspace = true }tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] }+tokio-retry = { workspace = true }tokio-util = { workspace = true, features = ["rt"] }tower = { workspace = true }tracing = { workspace = true }diff --git a/services/agent_harness_service/src/api.rs b/services/agent_harness_service/src/api.rsindex 932989baea..4b256831c1 100644--- a/services/agent_harness_service/src/api.rs+++ b/services/agent_harness_service/src/api.rs@@ -7,8 +7,6 @@ use std::time::Duration;use agent_egress::domain::service::EgressService;use agent_egress::inbound::axum_router::{EgressRouterState, egress_router};-use agent_harness::domain::service::ForwardedCommands;-use agent_harness::inbound::forward::{ForwardGatewayState, forward_router};use agent_harness::inbound::runtime_gateway::{RuntimeGatewayState, runtime_gateway_router};use agent_session::domain::ports::{AgentSessionNotificationRecipient, BotDirectory, SessionOpener,@@ -21,6 +19,8 @@ use agent_session::inbound::axum_router::{};use anyhow::Context;use axum::Router;+use axum::extract::State;+use axum::http::StatusCode;use axum::routing::get;use entity_access::domain::ports::EntityAccessService;use macro_authorization::MacroAuthorizationService;@@ -43,8 +43,8 @@ fn mount_at_root_and_prefix(inner: Router) -> Router {.nest(GATEWAY_PATH_PREFIX, inner)}-fn health_router() -> Router {- Router::new().route("/health", get(health))+fn health_router(ready: tokio::sync::watch::Receiver<bool>) -> Router {+ Router::new().route("/health", get(health).with_state(ready))}/// Serve the sandbox-facing egress proxy on its own listener.@@ -74,12 +74,12 @@ where}/// Build the router and serve it until the process is asked to stop.-pub async fn setup_and_serve<T, R, Opener, Bots, Access, Auth, Harness>(+pub async fn setup_and_serve<T, R, Opener, Bots, Access, Auth>(read_state: AgentSessionRouterState<T, Access, Auth>,control_state: AgentSessionControlState<R, Access, Auth>,create_state: CreateSessionState<Opener, Bots, Auth>,gateway_state: RuntimeGatewayState<Auth>,- forward_state: ForwardGatewayState<Harness, Auth>,+ runtime_commands_ready: tokio::sync::watch::Receiver<bool>,port: u16,shutdown: impl Future<Output = ()> + Send + 'static,) -> anyhow::Result<()>@@ -90,18 +90,11 @@ whereBots: BotDirectory,Access: EntityAccessService,Auth: MacroAuthorizationService,- Harness: ForwardedCommands,{- let inner = api_router(- read_state,- control_state,- create_state,- gateway_state,- forward_state,- )- .layer(MacroRequestIdAndTracingLayer::new(Duration::from_millis(200)).into_inner())- .merge(health_router())- .layer(macro_cors::cors_layer());+ let inner = api_router(read_state, control_state, create_state, gateway_state)+ .layer(MacroRequestIdAndTracingLayer::new(Duration::from_millis(200)).into_inner())+ .merge(health_router(runtime_commands_ready))+ .layer(macro_cors::cors_layer());let app = mount_at_root_and_prefix(inner).merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi())).merge(SwaggerUi::new("/agent-harness/docs").url(@@ -121,12 +114,11 @@ where.context("agent harness service http failed")}-fn api_router<T, R, Opener, Bots, Access, Auth, Harness>(+fn api_router<T, R, Opener, Bots, Access, Auth>(read_state: AgentSessionRouterState<T, Access, Auth>,control_state: AgentSessionControlState<R, Access, Auth>,create_state: CreateSessionState<Opener, Bots, Auth>,gateway_state: RuntimeGatewayState<Auth>,- forward_state: ForwardGatewayState<Harness, Auth>,) -> RouterwhereT: AgentSessionService,@@ -135,7 +127,6 @@ whereBots: BotDirectory,Access: EntityAccessService,Auth: MacroAuthorizationService,- Harness: ForwardedCommands,{let agent_sessions = agent_session_read_router(read_state.clone()).merge(agent_session_control_router(control_state))@@ -144,13 +135,12 @@ where.nest("/agent-sessions", agent_sessions).merge(agent_sandbox_size_router(read_state)).nest("/runtime", runtime_gateway_router(gateway_state))- // Replica-to-replica command forwarding. Internal-key authenticated,- // and reached over task-to-task networking rather than the load- // balancer; mounting it on the public listener is fine because the- // extractor refuses anything without the deployment's internal key.- .nest("/internal", forward_router(forward_state))}-async fn health() -> &'static str {- "ok"+async fn health(State(ready): State<tokio::sync::watch::Receiver<bool>>) -> StatusCode {+ if *ready.borrow() {+ StatusCode::OK+ } else {+ StatusCode::SERVICE_UNAVAILABLE+ }}diff --git a/services/agent_harness_service/src/api/test.rs b/services/agent_harness_service/src/api/test.rsindex cd34517c60..c97928b719 100644--- a/services/agent_harness_service/src/api/test.rs+++ b/services/agent_harness_service/src/api/test.rs@@ -8,7 +8,7 @@ use tower::ServiceExt;#[tokio::test]async fn health_is_reachable_at_root_and_gateway_prefix() {for path in ["/health", "/agent-harness/health"] {- let response = mount_at_root_and_prefix(health_router())+ let response = mount_at_root_and_prefix(health_router(tokio::sync::watch::channel(true).1)).oneshot(Request::builder().uri(path)@@ -25,7 +25,7 @@ async fn health_is_reachable_at_root_and_gateway_prefix() {#[tokio::test]async fn unprefixed_unknown_path_is_not_rewritten_onto_the_prefix() {- let response = mount_at_root_and_prefix(health_router())+ let response = mount_at_root_and_prefix(health_router(tokio::sync::watch::channel(true).1)).oneshot(Request::builder().uri("/missing")@@ -38,3 +38,19 @@ async fn unprefixed_unknown_path_is_not_rewritten_onto_the_prefix() {assert_eq!(response.status(), StatusCode::NOT_FOUND);}++#[tokio::test]+async fn health_fails_while_the_command_bus_is_disconnected() {+ let response = health_router(tokio::sync::watch::channel(false).1)+ .oneshot(+ Request::builder()+ .uri("/health")+ .method("GET")+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();++ assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);+}diff --git a/services/agent_harness_service/src/config.rs b/services/agent_harness_service/src/config.rsindex 7079b02f84..36c7f63513 100644--- a/services/agent_harness_service/src/config.rs+++ b/services/agent_harness_service/src/config.rs@@ -2,7 +2,7 @@//! `macro_config` pattern.use anyhow::Context;-use database_env_vars::DatabaseUrl;+use database_env_vars::{DatabaseUrl, RedisUri};pub use macro_env::Environment;use macro_uuid::Uuid;@@ -51,6 +51,8 @@ pub struct Config {pub kafka_brokers: KafkaBrokers,/// MacroDB connection string; `agent_sessions` lives here.pub database_url: DatabaseUrl,+ /// Shared Redis used for cross-replica command forwarding.+ pub redis_uri: RedisUri,/// Base URL of the Daytona REST API.#[macro_config_default(String::from("https://app.daytona.io/api"))]pub daytona_api_url: String,diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rsindex 597a00da70..228ee3312d 100644--- a/services/agent_harness_service/src/main.rs+++ b/services/agent_harness_service/src/main.rs@@ -13,6 +13,7 @@ mod bots_directory;mod config;mod containers;mod harness_bindings;+mod runtime_commands;mod trigger;use std::{future::Future, pin::Pin, sync::Arc};@@ -32,7 +33,6 @@ use agent_harness::domain::service::AgentHarnessService;use agent_harness::domain::trigger_router::{RoutedTrigger, agent_trigger_bot_id, route_agent_trigger,};-use agent_harness::inbound::forward::ForwardGatewayState;use agent_harness::inbound::runtime_gateway::RuntimeGatewayState;use agent_harness::outbound::agent_prompt_composer::LexicalAgentPromptComposer;use agent_harness::outbound::channel_announcer::ChannelAnnouncer;@@ -44,7 +44,7 @@ use agent_harness::outbound::daytona::{DaytonaContainerManager, DaytonaSettings, Snapshot,};use agent_harness::outbound::egress::EgressProvisioner;-use agent_harness::outbound::forward::HttpCommandForwarder;+use agent_harness::outbound::forward::RedisCommandForwarder;use agent_harness::outbound::local::{LocalContainerManager, LocalSettings};use agent_harness::outbound::routing::RoutedContainerManager;use agent_harness::outbound::runtime_registry::{HarnessKeyedConnections, RuntimeRegistry};@@ -52,7 +52,7 @@ use agent_inmem::outbound::log_frames::LogFrameSource;use agent_inmem::outbound::manager::InMemAgentManager;use agent_inmem::outbound::rig_engine::RigTurnEngine;use agent_runtime_directory::PgAgentRuntimeDirectory;-use agent_session::domain::model::{ReplicaAddress, ReplicaId};+use agent_session::domain::model::ReplicaId;use agent_session::domain::ports::{NoOpRealtime, SessionOwnership as _};use agent_session::domain::service::AgentSessionServiceImpl;use agent_session::inbound::axum_router::{@@ -102,8 +102,11 @@ use pipedream_mcp::outbound::pg_connection_repo::PgConnectionRepo;use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};use sqlx::postgres::PgPoolOptions;+use tokio_retry::{Retry, strategy::FixedInterval};use tracing::Instrument as _;+use runtime_commands::consume_runtime_commands;+/// Consumer group owning this harness's agent-session offsets.////// TODO: one group per bot deployment. Two bots sharing this name would@@ -111,6 +114,8 @@ use tracing::Instrument as _;/// exactly one harness deployment exists.struct AgentHarnessConsumerGroup;+const RUNTIME_COMMAND_CONSUMER_ATTEMPTS: usize = 5;+impl GroupName for AgentHarnessConsumerGroup {const GROUP_NAME: &'static str = "agent-harness-service";}@@ -137,63 +142,6 @@ struct PendingHarnessWork {description: &'static str,}-/// This task's own private base URL, for peers forwarding session commands.-///-/// On ECS the task metadata endpoint (v4) publishes the task's private IPv4;-/// the port is the service listener peers can reach through the task-to-task-/// security group rule. Local development has no routable address and runs-/// one replica, so `None` is correct there; every other environment refuses-/// to boot rather than heartbeat an unreachable identity, because a replica-/// without an address is one whose sessions peers can see but never reach.-async fn discover_replica_address(- environment: Environment,- port: u16,-) -> anyhow::Result<Option<agent_session::domain::model::ReplicaAddress>> {- // Explicit override first: how a local multi-replica experiment names- // each process (`REPLICA_ADDRESS=http://127.0.0.1:<port>`), since nothing- // off ECS can discover an address.- if let Some(address) = macro_env_var::optional_read_env_var("REPLICA_ADDRESS")- .ok()- .flatten()- {- return Ok(Some(agent_session::domain::model::ReplicaAddress::new(- address,- )));- }- let Some(metadata_uri) = macro_env_var::optional_read_env_var("ECS_CONTAINER_METADATA_URI_V4")- .ok()- .flatten()- else {- if matches!(environment, Environment::Local) {- return Ok(None);- }- anyhow::bail!(- "ECS_CONTAINER_METADATA_URI_V4 is unset outside a local environment: \- this replica would be unreachable for command forwarding"- );- };- let task: serde_json::Value = reqwest::Client::new()- .get(format!("{metadata_uri}/task"))- .timeout(std::time::Duration::from_secs(5))- .send()- .await- .context("failed to reach the ECS task metadata endpoint")?- .json()- .await- .context("failed to parse ECS task metadata")?;- let ip = task["Containers"]- .as_array()- .into_iter()- .flatten()- .flat_map(|container| container["Networks"].as_array().into_iter().flatten())- .flat_map(|network| network["IPv4Addresses"].as_array().into_iter().flatten())- .find_map(serde_json::Value::as_str)- .context("ECS task metadata carries no private IPv4 address")?;- Ok(Some(agent_session::domain::model::ReplicaAddress::new(- format!("http://{ip}:{port}"),- )))-}-#[tokio::main]async fn main() -> anyhow::Result<()> {let entrypoint = MacroEntrypoint::default().init();@@ -246,18 +194,10 @@ async fn run() -> anyhow::Result<()> {// log and pushes each frame at the channel's participants so a viewer sees// it happen.let session_repo = PgAgentSessionRepo::new(pool.clone());- // One lease identity for the whole process: commands forward to an- // address, and both attach-capable service instances below live at this- // one, so they share the id. The address is this task's own private URL,- // discovered from ECS metadata; a deployment that cannot learn it must- // not scale, so discovery failing anywhere but local refuses to boot.+ // One ownership identity for the whole process. Both attach-capable+ // service instances below live here, so they share it.let replica = ReplicaId::mint();- let replica_address = discover_replica_address(config.environment, config.port).await?;- tracing::info!(- %replica,- address = ?replica_address.as_ref().map(ReplicaAddress::as_str),- "harness replica identity"- );+ tracing::info!(%replica, "harness replica identity");// Bound to the harness once it exists (it is built *from* this service);// both attach-capable service instances report turns to the same one.let turn_observer = Arc::new(agent_session::domain::ports::LateBoundTurnObserver::new());@@ -466,6 +406,8 @@ async fn run() -> anyhow::Result<()> {// it and the harness takes sessions out of it. Attach/detach is mirrored// to the harnesses table so the settings page can show connection state.let runtimes = RuntimeRegistry::with_presence(Arc::new(PgHarnessPresence::new(pool.clone())));+ let redis = redis::Client::open(config.redis_uri.as_ref())+ .context("failed to create the runtime command Redis client")?;let mut defaults = HarnessDefaults::new(SessionDefaults {bot_id,model: config.harness_model.clone(),@@ -518,13 +460,50 @@ async fn run() -> anyhow::Result<()> {prompt_context,prompt_composer,EgressProvisioner::new(Arc::clone(&mcp_connections), config.egress_base_url.clone()),- HttpCommandForwarder::new(config.internal_api_key.clone())- .map_err(|error| anyhow::anyhow!("failed to build the command forwarder: {error}"))?,+ RedisCommandForwarder::new(redis.clone()),defaults,));// Close the loop: turn ends observed by the session actors drain the// harness's prompt queue.turn_observer.bind(harness.clone());+ let runtime_command_redis = redis.clone();+ let runtime_command_harness = harness.clone();+ let runtime_command_runtimes = Arc::clone(&runtimes);+ let (runtime_commands_ready, mut runtime_commands_readiness) =+ tokio::sync::watch::channel(false);+ let runtime_commands = tokio::spawn(async move {+ let result = Retry::start(+ FixedInterval::new(std::time::Duration::from_secs(1))+ .take(RUNTIME_COMMAND_CONSUMER_ATTEMPTS - 1),+ || {+ runtime_commands_ready.send_replace(false);+ consume_runtime_commands(+ runtime_command_redis.clone(),+ replica,+ {+ let runtimes = Arc::clone(&runtime_command_runtimes);+ Arc::new(move |harness| runtimes.is_connected(harness))+ },+ runtime_command_harness.clone(),+ runtime_commands_ready.clone(),+ )+ },+ )+ .await;+ if let Err(error) = result {+ tracing::error!(+ error = ?error,+ "runtime command Redis consumer stopped after five attempts"+ );+ }+ });+ tokio::time::timeout(+ std::time::Duration::from_secs(30),+ runtime_commands_readiness.wait_for(|ready| *ready),+ )+ .await+ .context("timed out subscribing to the runtime command bus")?+ .context("runtime command bus stopped before subscribing")?;// The complete session API is served from this process because it owns the// live sessions. Spawned rather than awaited: the Kafka loop below owns the@@ -567,10 +546,7 @@ async fn run() -> anyhow::Result<()> {runtimes,MacroAuthorizationState::new(Arc::new(authorization_service.clone())),);- let forward_state = ForwardGatewayState::new(- harness.clone(),- MacroAuthorizationState::new(Arc::new(authorization_service)),- );+ let http_runtime_commands_readiness = runtime_commands_readiness.clone();let http_port = config.port;let http = tokio::spawn(async move {if let Err(error) = api::setup_and_serve(@@ -578,7 +554,7 @@ async fn run() -> anyhow::Result<()> {control_state,create_state,gateway_state,- forward_state,+ http_runtime_commands_readiness,http_port,shutdown_signal(),)@@ -594,16 +570,16 @@ async fn run() -> anyhow::Result<()> {// release each claim eagerly in the actor teardown path; this loop is// what covers the ungraceful ones.let heartbeat_repo = session_repo.clone();- let heartbeat_address = replica_address.clone();+ let heartbeat_readiness = runtime_commands_readiness.clone();let heartbeat = tokio::spawn(async move {let mut ticker =tokio::time::interval(agent_session::domain::ports::REPLICA_HEARTBEAT_INTERVAL);loop {ticker.tick().await;- if let Err(error) = heartbeat_repo- .heartbeat(replica, heartbeat_address.as_ref())- .await- {+ if !*heartbeat_readiness.borrow() {+ continue;+ }+ if let Err(error) = heartbeat_repo.heartbeat(replica, None).await {tracing::warn!(error = ?error, %replica, "failed to heartbeat harness replica");}}@@ -837,6 +813,7 @@ async fn run() -> anyhow::Result<()> {trigger.abort();egress_http.abort();heartbeat.abort();+ runtime_commands.abort();let stop_failures = container_shutdown.shutdown_all().await;if stop_failures > 0 {tracing::error!(stop_failures, "some sandboxes failed to stop");diff --git a/services/agent_harness_service/src/runtime_commands.rs b/services/agent_harness_service/src/runtime_commands.rsnew file mode 100644index 0000000000..2e710ff6e7--- /dev/null+++ b/services/agent_harness_service/src/runtime_commands.rs@@ -0,0 +1,93 @@+//! Redis command-bus listener composed around the harness domain service.++use std::sync::Arc;++use agent_harness::domain::service::ForwardedCommands;+use agent_harness::outbound::forward::{+ COMMAND_CHANNEL, RuntimeCommandRequest, RuntimeCommandTarget,+};+use agent_session::domain::model::ReplicaId;+use futures::StreamExt as _;+use harness_id::HarnessId;+use redis::AsyncCommands as _;+use redis::{ExistenceCheck, SetExpiry, SetOptions};+use tracing::Instrument as _;++#[cfg(test)]+mod test;++pub(crate) async fn consume_runtime_commands<Harness>(+ redis: redis::Client,+ replica: ReplicaId,+ connected: Arc<dyn Fn(HarnessId) -> bool + Send + Sync>,+ harness_service: Arc<Harness>,+ ready: tokio::sync::watch::Sender<bool>,+) -> anyhow::Result<()>+where+ Harness: ForwardedCommands,+{+ let mut subscriber = redis.get_async_pubsub().await?;+ subscriber.subscribe(COMMAND_CHANNEL).await?;+ ready.send_replace(true);+ let mut requests = subscriber.into_on_message();+ while let Some(request) = requests.next().await {+ let payload: String = match request.get_payload() {+ Ok(payload) => payload,+ Err(error) => {+ tracing::warn!(error = ?error, "dropping malformed runtime command broadcast");+ continue;+ }+ };+ let request: RuntimeCommandRequest = match serde_json::from_str(&payload) {+ Ok(request) => request,+ Err(error) => {+ tracing::warn!(error = ?error, "dropping malformed runtime command broadcast");+ continue;+ }+ };+ let redis = redis.clone();+ let harness_service = Arc::clone(&harness_service);+ let connected = Arc::clone(&connected);+ let span = request.processing_span();+ tokio::spawn(+ async move {+ let request_id = request.request_id();+ let session = request.session();+ let selected = match request.target() {+ RuntimeCommandTarget::Replica(target) => target == replica.as_uuid(),+ RuntimeCommandTarget::Harness(harness) if connected(harness) => {+ let mut connection = match redis.get_multiplexed_async_connection().await {+ Ok(connection) => connection,+ Err(error) => {+ tracing::error!(error = ?error, "failed to claim a harness command");+ return;+ }+ };+ connection+ .set_options::<_, _, Option<String>>(+ format!("agent-harness.runtime-command.claim.{request_id}"),+ replica.as_uuid().to_string(),+ SetOptions::default()+ .conditional_set(ExistenceCheck::NX)+ .with_expiration(SetExpiry::EX(120)),+ )+ .await+ .is_ok_and(|result| result.is_some())+ }+ RuntimeCommandTarget::Harness(_) => false,+ };+ if selected {+ harness_service+ .execute_forwarded(session, request.into_command())+ .await+ .inspect_err(|error| {+ tracing::error!(error = ?error, %session, "forwarded command failed");+ })+ .ok();+ }+ }+ .instrument(span),+ );+ }+ anyhow::bail!("runtime command Redis subscription ended")+}diff --git a/services/agent_harness_service/src/runtime_commands/test.rs b/services/agent_harness_service/src/runtime_commands/test.rsnew file mode 100644index 0000000000..56eefb5158--- /dev/null+++ b/services/agent_harness_service/src/runtime_commands/test.rs@@ -0,0 +1,214 @@+use super::*;+use agent_harness::domain::error::{HarnessError, Result};+use agent_harness::domain::model::{CommandOutcome, HarnessCommand};+use agent_harness::domain::ports::{CommandForwarder as _, CommandTarget};+use agent_harness::outbound::forward::RedisCommandForwarder;+use agent_session::domain::model::AgentSessionId;+use std::sync::atomic::{AtomicUsize, Ordering};++struct RecordingHarness {+ calls: AtomicUsize,+ fails: bool,+}++impl ForwardedCommands for RecordingHarness {+ async fn execute_forwarded(+ &self,+ session_id: AgentSessionId,+ _command: HarnessCommand,+ ) -> Result<CommandOutcome> {+ self.calls.fetch_add(1, Ordering::SeqCst);+ if self.fails {+ return Err(HarnessError::Disconnected(session_id));+ }+ Ok(CommandOutcome::Completed)+ }+}++fn redis_client() -> redis::Client {+ let url = macro_env_var::optional_read_env_var("REDIS_URI")+ .ok()+ .flatten()+ .unwrap_or_else(|| "redis://127.0.0.1:6379".to_owned());+ redis::Client::open(url).expect("valid Redis URL")+}++async fn ready(readiness: &mut tokio::sync::watch::Receiver<bool>) {+ tokio::time::timeout(+ std::time::Duration::from_secs(2),+ readiness.wait_for(|ready| *ready),+ )+ .await+ .expect("consumer becomes ready")+ .expect("readiness channel remains open");+}++async fn wait_for_calls(harness: &RecordingHarness, expected: usize) {+ tokio::time::timeout(std::time::Duration::from_secs(2), async {+ while harness.calls.load(Ordering::SeqCst) != expected {+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("command is processed");+}++fn harness(fails: bool) -> Arc<RecordingHarness> {+ Arc::new(RecordingHarness {+ calls: AtomicUsize::new(0),+ fails,+ })+}++fn harness_id() -> HarnessId {+ HarnessId::new_from_uuid(macro_uuid::generate_uuid_v7())+}++#[tokio::test]+async fn a_command_executes_on_exactly_one_responsible_replica() {+ let redis = redis_client();+ let origin = ReplicaId::mint();+ let peer = ReplicaId::mint();+ let origin_harness = harness(false);+ let peer_harness = harness(false);+ let (origin_ready, mut origin_readiness) = tokio::sync::watch::channel(false);+ let origin_consumer = tokio::spawn(consume_runtime_commands(+ redis.clone(),+ origin,+ Arc::new(|_| false),+ Arc::clone(&origin_harness),+ origin_ready,+ ));+ let (peer_ready, mut peer_readiness) = tokio::sync::watch::channel(false);+ let peer_consumer = tokio::spawn(consume_runtime_commands(+ redis.clone(),+ peer,+ Arc::new(|_| false),+ Arc::clone(&peer_harness),+ peer_ready,+ ));+ ready(&mut origin_readiness).await;+ ready(&mut peer_readiness).await;++ let outcome = RedisCommandForwarder::new(redis)+ .forward(+ AgentSessionId::TEST_A,+ HarnessCommand::Delete,+ CommandTarget::Replica(peer),+ )+ .await+ .unwrap();++ assert_eq!(outcome, CommandOutcome::Completed);+ wait_for_calls(&peer_harness, 1).await;+ assert_eq!(origin_harness.calls.load(Ordering::SeqCst), 0);+ origin_consumer.abort();+ peer_consumer.abort();+}++#[tokio::test]+async fn a_responsible_replicas_error_does_not_fail_the_publish() {+ let redis = redis_client();+ let replica = ReplicaId::mint();+ let harness = harness(true);+ let (ready_tx, mut readiness) = tokio::sync::watch::channel(false);+ let consumer = tokio::spawn(consume_runtime_commands(+ redis.clone(),+ replica,+ Arc::new(|_| false),+ Arc::clone(&harness),+ ready_tx,+ ));+ ready(&mut readiness).await;++ let outcome = RedisCommandForwarder::new(redis)+ .forward(+ AgentSessionId::TEST_A,+ HarnessCommand::Delete,+ CommandTarget::Replica(replica),+ )+ .await+ .expect("the command publishes");++ assert_eq!(outcome, CommandOutcome::Completed);+ wait_for_calls(&harness, 1).await;+ consumer.abort();+}++#[tokio::test]+async fn publishing_without_an_owner_still_succeeds() {+ let redis = redis_client();+ let harness = harness(false);+ let (ready_tx, mut readiness) = tokio::sync::watch::channel(false);+ let consumer = tokio::spawn(consume_runtime_commands(+ redis.clone(),+ ReplicaId::mint(),+ Arc::new(|_| false),+ Arc::clone(&harness),+ ready_tx,+ ));+ ready(&mut readiness).await;++ let target = harness_id();+ let outcome = RedisCommandForwarder::new(redis)+ .forward(+ AgentSessionId::TEST_A,+ HarnessCommand::Delete,+ CommandTarget::Harness(target),+ )+ .await+ .expect("the command publishes");++ assert_eq!(outcome, CommandOutcome::Completed);+ assert_eq!(harness.calls.load(Ordering::SeqCst), 0);+ consumer.abort();+}++#[tokio::test]+async fn overlapping_harness_connections_execute_once() {+ let redis = redis_client();+ let first = harness(false);+ let second = harness(false);+ let target = harness_id();+ let (first_ready, mut first_readiness) = tokio::sync::watch::channel(false);+ let first_consumer = tokio::spawn(consume_runtime_commands(+ redis.clone(),+ ReplicaId::mint(),+ Arc::new(move |harness| harness == target),+ Arc::clone(&first),+ first_ready,+ ));+ let (second_ready, mut second_readiness) = tokio::sync::watch::channel(false);+ let second_consumer = tokio::spawn(consume_runtime_commands(+ redis.clone(),+ ReplicaId::mint(),+ Arc::new(move |harness| harness == target),+ Arc::clone(&second),+ second_ready,+ ));+ ready(&mut first_readiness).await;+ ready(&mut second_readiness).await;++ RedisCommandForwarder::new(redis)+ .forward(+ AgentSessionId::TEST_A,+ HarnessCommand::Delete,+ CommandTarget::Harness(target),+ )+ .await+ .unwrap();++ tokio::time::timeout(std::time::Duration::from_secs(2), async {+ while first.calls.load(Ordering::SeqCst) + second.calls.load(Ordering::SeqCst) != 1 {+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("command is processed");+ assert_eq!(+ first.calls.load(Ordering::SeqCst) + second.calls.load(Ordering::SeqCst),+ 1+ );+ first_consumer.abort();+ second_consumer.abort();+}