Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(agent-harness): route commands through Redis (#6195)

Wolf Mermelstein48a31c227652
diff --git a/Cargo.lock b/Cargo.lock
index d2e4f6fe51..dfe02fedda 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -264,7 +264,9 @@ dependencies = [
"macro_uuid",
"mockall",
"model_user",
+ "opentelemetry",
"pipedream_mcp",
+ "redis",
"reqwest 0.13.4",
"rootcause",
"rustls 0.23.40",
@@ -276,6 +278,7 @@ dependencies = [
"tokio-tungstenite 0.28.0",
"tokio-util",
"tracing",
+ "tracing-opentelemetry",
"url",
"workspace-hack",
]
@@ -307,6 +310,7 @@ dependencies = [
"cursor_cloud_agents",
"database_env_vars",
"entity_access",
+ "futures",
"github",
"harness_id",
"kafka_util",
@@ -328,13 +332,14 @@ dependencies = [
"notification",
"pipedream_mcp",
"rdkafka",
- "reqwest 0.13.4",
+ "redis",
"rootcause",
"secretsmanager_client",
"serde",
"serde_json",
"sqlx",
"tokio",
+ "tokio-retry",
"tokio-util",
"tower 0.5.3",
"tracing",
diff --git a/crates/agent_harness/Cargo.toml b/crates/agent_harness/Cargo.toml
index f1f07cd120..351a6071df 100644
--- a/crates/agent_harness/Cargo.toml
+++ b/crates/agent_harness/Cargo.toml
@@ -34,7 +34,9 @@ macro_authorization = { path = "../macro_authorization", features = ["axum"] }
macro_tower_layers = { path = "../macro_tower_layers" }
macro_user_id = { path = "../macro_user_id" }
macro_uuid = { path = "../macro_uuid" }
+opentelemetry = { workspace = true, features = ["trace"] }
reqwest = { workspace = true }
+redis = { workspace = true, features = ["aio", "tokio-comp"] }
rootcause = { workspace = true }
rustls = { workspace = true }
serde = { workspace = true }
@@ -51,6 +53,7 @@ tokio = { workspace = true, features = [
tokio-util = { workspace = true, features = ["rt"] }
tokio-tungstenite = { workspace = true }
tracing = { workspace = true }
+tracing-opentelemetry = { workspace = true }
url = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/agent_harness/src/domain/error.rs b/crates/agent_harness/src/domain/error.rs
index b255752516..9a2105cfeb 100644
--- a/crates/agent_harness/src/domain/error.rs
+++ b/crates/agent_harness/src/domain/error.rs
@@ -63,11 +63,6 @@ pub enum HarnessError {
/// A prompt could not be composed for the agent runtime.
#[error("failed to compose agent prompt: {0}")]
PromptComposition(rootcause::Report),
- /// A live peer manages the session but published no address to forward
- /// to. Transient by construction: only a replica from before addresses
- /// existed, or one whose first heartbeat has not landed, has no address.
- #[error("agent session {0} is managed by a live replica with no forwarding address")]
- ManagerUnreachable(AgentSessionId),
/// Forwarding a command to the session's managing replica failed.
#[error("failed to forward an agent session command: {0}")]
Forward(rootcause::Report),
diff --git a/crates/agent_harness/src/domain/ports.rs b/crates/agent_harness/src/domain/ports.rs
index 0f8416259b..916d636145 100644
--- a/crates/agent_harness/src/domain/ports.rs
+++ b/crates/agent_harness/src/domain/ports.rs
@@ -5,7 +5,7 @@ use std::pin::Pin;
use std::sync::Arc;
use agent_session::domain::connection::RuntimeAttachment;
-use agent_session::domain::model::{AgentSessionId, ReplicaAddress, SandboxSize};
+use agent_session::domain::model::{AgentSessionId, SandboxSize};
use agent_session::domain::ports::AgentConnector;
use bot_id::BotId;
use harness_id::HarnessId;
@@ -19,22 +19,23 @@ use super::model::{
};
use super::sandbox::SandboxResizeEffect;
-/// Delivers a session's command to the replica that manages its live actor.
-///
-/// The receiving side executes without re-resolving management (a forward is
-/// single-hop by contract, so two replicas with momentarily different views
-/// cannot bounce a command between each other). Success means the peer ran
-/// the command to completion - the response is the acknowledgment - so a
-/// caller that awaited a forward has the same guarantee as one that executed
-/// locally.
+/// The distributed destination for a forwarded command.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CommandTarget {
+ /// The replica that owns the session actor.
+ Replica(agent_session::domain::model::ReplicaId),
+ /// The replica holding a registered harness's runtime socket.
+ Harness(HarnessId),
+}
+
+/// Forwards commands to the replica currently responsible for execution.
pub trait CommandForwarder: Send + Sync + 'static {
- /// Run `command` for `session` on the replica at `target`, reporting
- /// what that replica's execution did with it.
+ /// Run `command` at `target`.
fn forward(
&self,
- target: &ReplicaAddress,
session: AgentSessionId,
command: HarnessCommand,
+ target: CommandTarget,
) -> impl Future<Output = Result<CommandOutcome>> + Send;
}
@@ -47,13 +48,11 @@ pub struct NoPeers;
impl CommandForwarder for NoPeers {
async fn forward(
&self,
- target: &ReplicaAddress,
session: AgentSessionId,
_command: HarnessCommand,
+ _target: CommandTarget,
) -> Result<CommandOutcome> {
- Err(HarnessError::Forward(rootcause::report!(
- "this deployment has no command forwarding, yet {target} manages session {session}"
- )))
+ Err(HarnessError::Disconnected(session))
}
}
@@ -170,6 +169,9 @@ pub trait RuntimeConnections: Send + Sync + 'static {
&self,
bot: BotId,
) -> impl Future<Output = anyhow::Result<Option<HarnessId>>> + Send;
+
+ /// Whether this process currently holds `harness`'s physical runtime socket.
+ fn is_connected(&self, harness: HarnessId) -> bool;
}
/// Mints the one secret a sandbox is given, and the config that points it at
diff --git a/crates/agent_harness/src/domain/service.rs b/crates/agent_harness/src/domain/service.rs
index 6a5eb9bf45..b6ff47a4ee 100644
--- a/crates/agent_harness/src/domain/service.rs
+++ b/crates/agent_harness/src/domain/service.rs
@@ -59,20 +59,20 @@ struct QueuedCommand {
trait ErasedForwarder: Send + Sync + 'static {
fn forward<'a>(
&'a self,
- target: &'a agent_session::domain::model::ReplicaAddress,
session: AgentSessionId,
command: HarnessCommand,
+ target: crate::domain::ports::CommandTarget,
) -> std::pin::Pin<Box<dyn Future<Output = Result<CommandOutcome>> + Send + 'a>>;
}
impl<F: CommandForwarder> ErasedForwarder for F {
fn forward<'a>(
&'a self,
- target: &'a agent_session::domain::model::ReplicaAddress,
session: AgentSessionId,
command: HarnessCommand,
+ target: crate::domain::ports::CommandTarget,
) -> std::pin::Pin<Box<dyn Future<Output = Result<CommandOutcome>> + Send + 'a>> {
- Box::pin(CommandForwarder::forward(self, target, session, command))
+ Box::pin(CommandForwarder::forward(self, session, command, target))
}
}
@@ -337,10 +337,10 @@ where
}
}
-/// The receiving half of command forwarding: what the internal forward route
-/// calls, implemented by the harness as [`AgentHarnessService::execute_here`].
+/// The receiving half of command forwarding, called by the command-bus
+/// consumer and implemented by the harness as [`AgentHarnessService::execute_here`].
pub trait ForwardedCommands: Send + Sync + 'static {
- /// Run a command a peer already routed to this replica.
+ /// Run a command the transport has targeted to this replica.
fn execute_forwarded(
&self,
session_id: AgentSessionId,
@@ -761,6 +761,7 @@ fn queue_result<T>(
fn into_session_error(error: HarnessError) -> AgentSessionError {
match error {
HarnessError::Session(error) => error,
+ HarnessError::Disconnected(session) => AgentSessionError::Disconnected(session),
other => AgentSessionError::Unknown(anyhow::anyhow!(other)),
}
}
@@ -786,12 +787,6 @@ where
{
/// Execute where the session's live actor is: locally when nobody (or
/// this replica) manages it, on the managing peer otherwise.
- ///
- /// A failed forward re-reads the lease once: the one legitimate reason a
- /// live manager refuses its own session is that it died mid-flight, and
- /// then its heartbeat going stale is what lets this replica take over. A
- /// manager that is alive but unreachable stays an error - executing
- /// locally anyway is how two actors end up on one session.
/// The routing decision is recorded on the span, not only logged: which of
/// the three answers the lease gave, which peer it named, and whether the
/// command left this process. Those are the fields you group by when a
@@ -804,7 +799,6 @@ where
agent.session.management = tracing::field::Empty,
agent.session.manager_replica = tracing::field::Empty,
agent.command.forwarded = tracing::field::Empty,
- agent.command.stale_fallback = tracing::field::Empty,
)
)]
async fn route_then_execute(
@@ -824,7 +818,30 @@ where
SessionManagement::Unmanaged => {
span.record("agent.session.management", "unmanaged");
span.record("agent.command.forwarded", false);
- return self.execute(session_id, command).await;
+ let session = self.sessions.get_session(session_id).await?;
+ if AgentKind::for_session(session.bot_id, &session.harness) != AgentKind::External {
+ return self.execute(session_id, command).await;
+ }
+ let Some(harness) = self
+ .runtimes
+ .bound_harness(session.bot_id)
+ .await
+ .map_err(AgentSessionError::Unknown)?
+ else {
+ return self.execute(session_id, command).await;
+ };
+ if self.runtimes.is_connected(harness) {
+ return self.execute(session_id, command).await;
+ }
+ span.record("agent.command.forwarded", true);
+ return self
+ .forwarder
+ .forward(
+ session_id,
+ command,
+ crate::domain::ports::CommandTarget::Harness(harness),
+ )
+ .await;
}
SessionManagement::Ours => {
span.record("agent.session.management", "ours");
@@ -838,36 +855,15 @@ where
"agent.session.manager_replica",
tracing::field::display(manager.replica),
);
- let Some(address) = manager.address else {
- // Recorded false deliberately: the command stayed here, but as an
- // error rather than a local execution.
- span.record("agent.command.forwarded", false);
- return Err(HarnessError::ManagerUnreachable(session_id));
- };
span.record("agent.command.forwarded", true);
tracing::info!(%session_id, peer = %manager.replica, "forwarding an agent session command");
- match self
- .forwarder
- .forward(&address, session_id, command.clone())
+ self.forwarder
+ .forward(
+ session_id,
+ command,
+ crate::domain::ports::CommandTarget::Replica(manager.replica),
+ )
.await
- {
- Ok(outcome) => Ok(outcome),
- Err(forward_error) => match self.sessions.management(session_id).await? {
- SessionManagement::Unmanaged | SessionManagement::Ours => {
- // Worth aggregating rather than only logging: routine
- // fallbacks mean heartbeats are not keeping up, which is a
- // different problem from an occasional dead peer.
- span.record("agent.command.stale_fallback", true);
- tracing::warn!(
- error = ?forward_error,
- %session_id,
- "the managing replica went stale mid-forward; executing locally"
- );
- self.execute(session_id, command).await
- }
- SessionManagement::Peer(_) => Err(forward_error),
- },
- }
}
async fn execute(
diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rs
index 90ef3edf5c..df2a33d888 100644
--- a/crates/agent_harness/src/domain/service/test.rs
+++ b/crates/agent_harness/src/domain/service/test.rs
@@ -31,6 +31,7 @@ use macro_uuid::Uuid;
use tokio::sync::mpsc;
use super::AgentHarnessService;
+use super::into_session_error;
use crate::domain::error::HarnessError;
use crate::domain::model::{
AgentKind, AgentRuntimeConfig, AnnounceOrigin, CommandOutcome, DeliverAction, HarnessCommand,
@@ -54,6 +55,14 @@ fn sender() -> MacroUserIdStr<'static> {
MacroUserIdStr::try_from_email("[email protected]").expect("a valid user id")
}
+#[test]
+fn disconnected_harness_errors_keep_the_session_error_classification() {
+ assert!(matches!(
+ into_session_error(HarnessError::Disconnected(AgentSessionId::TEST_A)),
+ AgentSessionError::Disconnected(AgentSessionId::TEST_A)
+ ));
+}
+
/// A sender whose email domain is `is_macro_staff` - the identity the
/// Daytona staff gate admits.
fn staff_sender() -> MacroUserIdStr<'static> {
@@ -1856,7 +1865,7 @@ async fn a_disconnected_external_session_never_gets_a_sandbox() {
assert!(
matches!(
error,
- HarnessError::Session(AgentSessionError::Disconnected(id)) if id == session.id,
+ HarnessError::Disconnected(id) if id == session.id,
),
"got {error:?}"
);
@@ -2147,78 +2156,51 @@ async fn set_sandbox_size_unsupported_does_not_persist() {
assert_eq!(containers.resumed(), 0);
}
+type ForwardCall = (crate::domain::ports::CommandTarget, AgentSessionId);
+
/// A [`CommandForwarder`] that records its calls and reports success.
#[derive(Clone, Default)]
struct RecordingForwarder {
- calls: Arc<Mutex<Vec<(String, AgentSessionId)>>>,
+ calls: Arc<Mutex<Vec<ForwardCall>>>,
}
impl crate::domain::ports::CommandForwarder for RecordingForwarder {
async fn forward(
&self,
- target: &agent_session::domain::model::ReplicaAddress,
session: AgentSessionId,
_command: HarnessCommand,
+ target: crate::domain::ports::CommandTarget,
) -> crate::domain::error::Result<CommandOutcome> {
- self.calls
- .lock()
- .unwrap()
- .push((target.as_str().to_owned(), session));
+ self.calls.lock().unwrap().push((target, session));
Ok(CommandOutcome::Completed)
}
}
-/// A forwarder standing in for a peer that died mid-forward: the call fails,
-/// and by the time the caller re-reads the lease the peer's claim is gone.
-#[derive(Clone)]
-struct DyingPeerForwarder {
- repo: InMemoryAgentSessionRepo,
- claim: agent_session::domain::model::SessionClaim,
-}
-
-impl crate::domain::ports::CommandForwarder for DyingPeerForwarder {
- async fn forward(
- &self,
- _target: &agent_session::domain::model::ReplicaAddress,
- _session: AgentSessionId,
- _command: HarnessCommand,
- ) -> crate::domain::error::Result<CommandOutcome> {
- use agent_session::domain::ports::SessionOwnership as _;
- self.repo.release(&self.claim).await.expect("peer releases");
- Err(HarnessError::Forward(rootcause::report!(
- "the peer went away"
- )))
- }
-}
-
-/// Claim a session for a fabricated peer replica that publishes an address.
+/// Claim a session for a fabricated peer replica.
async fn claim_as_peer(
repo: &InMemoryAgentSessionRepo,
session: AgentSessionId,
- address: &str,
) -> agent_session::domain::model::SessionClaim {
- use agent_session::domain::model::{ClaimOutcome, ReplicaAddress, ReplicaId};
+ use agent_session::domain::model::{ClaimOutcome, ReplicaId};
use agent_session::domain::ports::SessionOwnership as _;
let peer = ReplicaId::mint();
- repo.heartbeat(peer, Some(&ReplicaAddress::new(address)))
- .await
- .expect("peer heartbeats");
+ repo.heartbeat(peer, None).await.expect("peer heartbeats");
match repo.claim(session, peer).await.expect("peer claims") {
ClaimOutcome::Claimed(claim) => claim,
ClaimOutcome::ManagedElsewhere(_) => panic!("nobody else should hold the test session"),
}
}
-/// A command for a session a live peer manages goes to the peer's address
+/// A command for a session a live peer manages goes through the command bus
/// and never executes here: the session outlives a Delete this replica would
/// otherwise have applied.
#[tokio::test]
-async fn commands_for_a_peer_managed_session_forward_to_its_address() {
+async fn commands_for_a_peer_managed_session_forward_through_redis() {
let repo = InMemoryAgentSessionRepo::new();
let session = agent_session::testing::test_agent_session(AgentSessionId::new());
let id = session.id;
repo.insert_session(session);
- let _claim = claim_as_peer(&repo, id, "http://10.0.0.7:8100").await;
+ let claim = claim_as_peer(&repo, id).await;
let forwarder = RecordingForwarder::default();
let service = AgentHarnessService::new(
AgentSessionServiceImpl::new(
@@ -2246,26 +2228,23 @@ async fn commands_for_a_peer_managed_session_forward_to_its_address() {
.await
.expect("the forwarded command succeeds");
- assert_eq!(
- forwarder.calls.lock().unwrap().clone(),
- vec![("http://10.0.0.7:8100".to_owned(), id)]
- );
+ assert_eq!(forwarder.calls.lock().unwrap().len(), 1);
+ assert!(matches!(
+ forwarder.calls.lock().unwrap()[0],
+ (crate::domain::ports::CommandTarget::Replica(replica), session)
+ if replica == claim.replica && session == id
+ ));
assert!(
repo.get(id).await.is_ok(),
"the delete ran on the peer, not here"
);
}
-/// A forward that fails against a peer whose lease is gone by the re-read
-/// falls back to executing locally, so a peer crash costs one retry rather
-/// than the command.
#[tokio::test]
-async fn a_dead_peers_command_falls_back_to_local_execution() {
+async fn unmanaged_external_session_forwards_to_its_remote_harness() {
let repo = InMemoryAgentSessionRepo::new();
- let session = agent_session::testing::test_agent_session(AgentSessionId::new());
- let id = session.id;
- repo.insert_session(session);
- let claim = claim_as_peer(&repo, id, "http://10.0.0.7:8100").await;
+ let runtimes = TestConnections::new(MirrorBindings, RuntimeRegistry::new());
+ let forwarder = RecordingForwarder::default();
let service = AgentHarnessService::new(
AgentSessionServiceImpl::new(
repo.clone(),
@@ -2274,14 +2253,11 @@ async fn a_dead_peers_command_falls_back_to_local_execution() {
),
MockContainerManager::new(),
AnnouncerMock::new(),
- TestConnections::new(MirrorBindings, RuntimeRegistry::<ContainerSender>::new()),
+ runtimes,
PromptContextMock::default(),
PromptComposerMock::default(),
EgressProvisionerMock::new(),
- DyingPeerForwarder {
- repo: repo.clone(),
- claim,
- },
+ forwarder.clone(),
SessionDefaults {
bot_id: BotId::TEST_A,
model: "claude".to_owned(),
@@ -2289,14 +2265,25 @@ async fn a_dead_peers_command_falls_back_to_local_execution() {
repo_url: "https://github.com/macro-inc/macro".to_owned(),
},
);
+ let session = service
+ .open_external_session(open_external_request("/srv/agent"))
+ .await
+ .expect("external session opens without a local runtime");
service
- .execute(id, HarnessCommand::Delete)
+ .execute(session.id, HarnessCommand::Delete)
.await
- .expect("the fallback executes locally");
+ .expect("the remote harness executes the command");
+ assert_eq!(
+ *forwarder.calls.lock().unwrap(),
+ [(
+ crate::domain::ports::CommandTarget::Harness(harness_for_bot(session.bot_id)),
+ session.id,
+ )]
+ );
assert!(
- repo.get(id).await.is_err(),
- "the delete ran here once the peer was gone"
+ repo.get(session.id).await.is_ok(),
+ "the command did not execute on this replica"
);
}
diff --git a/crates/agent_harness/src/inbound/forward.rs b/crates/agent_harness/src/inbound/forward.rs
deleted file mode 100644
index 6093e9523a..0000000000
--- a/crates/agent_harness/src/inbound/forward.rs
+++ /dev/null
@@ -1,127 +0,0 @@
-//! The receiving half of replica-to-replica command forwarding.
-//!
-//! A peer that consumed a command for a session this replica manages POSTs it
-//! here. Internal-only by construction: the caller is another replica of this
-//! same deployment, authenticated with the shared internal key, never a user.
-//! The handler executes without re-resolving management - forwarding is
-//! single-hop, and this endpoint is the second hop honoring that contract.
-
-use std::sync::Arc;
-
-use agent_session::domain::model::AgentSessionId;
-use axum::Router;
-use axum::extract::{FromRef, Json, Path, State};
-use axum::http::StatusCode;
-use axum::response::{IntoResponse, Response};
-use axum::routing::post;
-use macro_authorization::{
- InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService, MacroAuthorizationState,
-};
-use macro_uuid::Uuid;
-
-use crate::domain::error::HarnessError;
-use crate::domain::model::HarnessCommand;
-use crate::domain::service::ForwardedCommands;
-
-/// State for the command-forwarding route.
-pub struct ForwardGatewayState<Harness, Auth> {
- harness: Arc<Harness>,
- authorization_state: MacroAuthorizationState<Auth>,
-}
-
-impl<Harness, Auth> ForwardGatewayState<Harness, Auth> {
- /// Create forwarding state.
- pub fn new(harness: Arc<Harness>, authorization_state: MacroAuthorizationState<Auth>) -> Self {
- Self {
- harness,
- authorization_state,
- }
- }
-}
-
-// Manual Clone impl: everything is behind an Arc.
-impl<Harness, Auth> Clone for ForwardGatewayState<Harness, Auth> {
- fn clone(&self) -> Self {
- Self {
- harness: Arc::clone(&self.harness),
- authorization_state: self.authorization_state.clone(),
- }
- }
-}
-
-impl<Harness, Auth> FromRef<ForwardGatewayState<Harness, Auth>> for MacroAuthorizationState<Auth> {
- fn from_ref(state: &ForwardGatewayState<Harness, Auth>) -> Self {
- state.authorization_state.clone()
- }
-}
-
-/// Build the router serving `POST /agent-sessions/{id}/command`. Mount it
-/// under `/internal`.
-pub fn forward_router<Harness, Auth, S>(state: ForwardGatewayState<Harness, Auth>) -> Router<S>
-where
- Harness: ForwardedCommands,
- Auth: MacroAuthorizationService,
- S: Clone + Send + Sync + 'static,
-{
- Router::new()
- .route(
- "/agent-sessions/{session_id}/command",
- post(forward_handler::<Harness, Auth>),
- )
- .with_state(state)
-}
-
-/// Execute a command a peer replica forwarded to this one.
-///
-/// Instrumented because this is the far half of the hop: the request carries
-/// the sending replica's `traceparent`, which the tracing layer turns into a
-/// parent link, so this span shows a forwarded command executing under the
-/// span that routed it instead of as an unrelated root. `outcome` distinguishes
-/// the three answers the sender can get without reading the response body.
-#[tracing::instrument(
- name = "harness.forward.receive",
- skip_all,
- fields(
- agent.session.id = %session_id,
- agent.command.forwarded = true,
- agent.command.outcome = tracing::field::Empty,
- )
-)]
-async fn forward_handler<Harness, Auth>(
- State(state): State<ForwardGatewayState<Harness, Auth>>,
- _caller: MacroAuthorizationExtractor<Auth, InternalOnly>,
- Path(session_id): Path<Uuid>,
- Json(command): Json<HarnessCommand>,
-) -> Response
-where
- Harness: ForwardedCommands,
- Auth: MacroAuthorizationService,
-{
- let session_id = AgentSessionId::new_from_uuid(session_id);
- let span = tracing::Span::current();
- match state.harness.execute_forwarded(session_id, command).await {
- // The outcome rides back in the body so the sender can answer its own
- // caller honestly - a queued deliver is not a delivered one.
- Ok(outcome) => (StatusCode::OK, axum::Json(outcome)).into_response(),
- // Surfaced with its own status so the sender can tell "the actor is
- // not here after all" from an execution failure: the sender's
- // fallback re-reads the lease on any error, but a 409 is the signal
- // that its view was stale rather than that the command is bad.
- Err(HarnessError::Disconnected(id))
- | Err(HarnessError::Session(
- agent_session::domain::error::AgentSessionError::Disconnected(id),
- )) => {
- span.record("agent.command.outcome", "not_attached_here");
- (
- StatusCode::CONFLICT,
- format!("session {id} is not attached here"),
- )
- .into_response()
- }
- Err(error) => {
- span.record("agent.command.outcome", "failed");
- tracing::error!(error = ?error, %session_id, "a forwarded command failed");
- (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response()
- }
- }
-}
diff --git a/crates/agent_harness/src/inbound/mod.rs b/crates/agent_harness/src/inbound/mod.rs
index aeac45aac8..856caaa13c 100644
--- a/crates/agent_harness/src/inbound/mod.rs
+++ b/crates/agent_harness/src/inbound/mod.rs
@@ -3,6 +3,5 @@
//! Thin by construction - decode the broker message, ask the domain whether it
//! is addressed to us, hand one value inward. No policy lives here.
-pub mod forward;
pub mod kafka;
pub mod runtime_gateway;
diff --git a/crates/agent_harness/src/outbound/forward.rs b/crates/agent_harness/src/outbound/forward.rs
index a408d9ce5f..9535bfb455 100644
--- a/crates/agent_harness/src/outbound/forward.rs
+++ b/crates/agent_harness/src/outbound/forward.rs
@@ -1,104 +1,148 @@
-//! HTTP implementation of [`CommandForwarder`]: one authenticated POST to the
-//! managing replica's internal forward route, awaited for its result.
-//!
-//! Direct replica-to-replica, not through the load balancer: the target
-//! address is the peer's own private URL as published with its heartbeat, so
-//! the command lands on exactly the process holding the session's actor.
+//! Redis implementation of [`CommandForwarder`].
-use agent_session::domain::model::{AgentSessionId, ReplicaAddress};
-use reqwest::StatusCode;
+use agent_session::domain::model::AgentSessionId;
+use harness_id::HarnessId;
+use redis::AsyncCommands as _;
+use std::collections::HashMap;
+use tracing_opentelemetry::OpenTelemetrySpanExt as _;
use crate::domain::error::{HarnessError, Result};
use crate::domain::model::{CommandOutcome, HarnessCommand};
-use crate::domain::ports::CommandForwarder;
+use crate::domain::ports::{CommandForwarder, CommandTarget};
-/// Header carrying the deployment's shared internal key, as
-/// `macro_authorization`'s internal extractor reads it.
-const INTERNAL_API_KEY_HEADER: &str = macro_authorization::INTERNAL_API_KEY_HEADER;
-
-/// Forwards commands to peer replicas over their internal HTTP surface.
+/// Broadcasts commands to the responsible harness replica over Redis.
#[derive(Clone)]
-pub struct HttpCommandForwarder {
- client: reqwest::Client,
- internal_api_key: String,
+pub struct RedisCommandForwarder {
+ redis: redis::Client,
}
-impl HttpCommandForwarder {
- /// A forwarder authenticating with the deployment's internal key.
- ///
- /// The timeout bounds the whole forwarded execution, not just the dial:
- /// the peer replies only once the command ran, and a Deliver can sit
- /// behind a sandbox resume, so this is generous rather than snappy.
- pub fn new(internal_api_key: String) -> Result<Self> {
- let client = reqwest::Client::builder()
- .connect_timeout(std::time::Duration::from_secs(2))
- .timeout(std::time::Duration::from_secs(120))
- .build()
- .map_err(|error| HarnessError::Forward(rootcause::report!(error).into_dynamic()))?;
- Ok(Self {
- client,
- internal_api_key,
- })
+impl RedisCommandForwarder {
+ /// Build a Redis command forwarder.
+ pub fn new(redis: redis::Client) -> Self {
+ Self { redis }
}
}
-impl CommandForwarder for HttpCommandForwarder {
- #[tracing::instrument(
- err,
- skip(self, command),
- fields(
- %target,
- %session,
- http.response.status_code = tracing::field::Empty,
- )
- )]
+impl CommandForwarder for RedisCommandForwarder {
async fn forward(
&self,
- target: &ReplicaAddress,
session: AgentSessionId,
command: HarnessCommand,
+ target: CommandTarget,
) -> Result<CommandOutcome> {
- let url = format!(
- "{}/internal/agent-sessions/{}/command",
- target.as_str().trim_end_matches('/'),
- session
- );
- // The peer's inbound layer already extracts `traceparent` and parents
- // its request span to the remote span (see `macro_tower_layers`), so
- // injecting here is what makes a forwarded command one trace spanning
- // both replicas. Without it the hop is two unrelated traces.
- let mut headers = reqwest::header::HeaderMap::new();
- macro_tower_layers::inject_trace_headers(&mut headers);
- let response = self
- .client
- .post(url)
- .header(INTERNAL_API_KEY_HEADER, &self.internal_api_key)
- .headers(headers)
- .json(&command)
- .send()
+ let request = RuntimeCommandRequest {
+ request_id: macro_uuid::Uuid::new_v4(),
+ target: RuntimeCommandTarget::from(target),
+ session,
+ command,
+ trace_context: current_trace_context(),
+ };
+ let payload = serde_json::to_string(&request)
+ .map_err(|error| forward_error("serialize command request", error))?;
+ let mut publisher = self
+ .redis
+ .get_multiplexed_async_connection()
.await
- .map_err(|error| HarnessError::Forward(rootcause::report!(error).into_dynamic()))?;
- let status = response.status();
- tracing::Span::current().record("http.response.status_code", status.as_u16());
- if status.is_success() {
- // A body-less success (an older peer's 204, mid-deploy) reads as
- // completed: that was the only outcome such a peer could produce.
- return Ok(response
- .json::<CommandOutcome>()
- .await
- .unwrap_or(CommandOutcome::Completed));
+ .map_err(|error| forward_error("open command publisher", error))?;
+ publisher
+ .publish::<_, _, ()>(COMMAND_CHANNEL, payload)
+ .await
+ .map_err(|error| forward_error("publish command request", error))?;
+ Ok(CommandOutcome::Completed)
+ }
+}
+
+/// Redis channel broadcasting runtime commands to every harness replica.
+pub const COMMAND_CHANNEL: &str = "agent-harness.runtime-commands";
+
+/// A command broadcast received from another replica.
+#[derive(serde::Serialize, serde::Deserialize)]
+pub struct RuntimeCommandRequest {
+ request_id: macro_uuid::Uuid,
+ target: RuntimeCommandTarget,
+ session: AgentSessionId,
+ command: HarnessCommand,
+ #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+ trace_context: HashMap<String, String>,
+}
+
+/// The process-local destination selected by the command sender.
+#[derive(Clone, Copy, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum RuntimeCommandTarget {
+ /// A specific session-owning replica.
+ Replica(macro_uuid::Uuid),
+ /// Whichever replica currently holds this harness runtime connection.
+ Harness(HarnessId),
+}
+
+impl RuntimeCommandRequest {
+ /// The unique ID used to claim and trace this command.
+ #[must_use]
+ pub fn request_id(&self) -> macro_uuid::Uuid {
+ self.request_id
+ }
+
+ /// The replica or harness connection selected for execution.
+ #[must_use]
+ pub fn target(&self) -> RuntimeCommandTarget {
+ self.target
+ }
+
+ /// The session receiving the command.
+ #[must_use]
+ pub fn session(&self) -> AgentSessionId {
+ self.session
+ }
+
+ /// Consume the transport envelope and return its domain command.
+ #[must_use]
+ pub fn into_command(self) -> HarnessCommand {
+ self.command
+ }
+
+ /// Create a consumer span parented to the sender's propagated context.
+ #[must_use]
+ pub fn processing_span(&self) -> tracing::Span {
+ runtime_command_span(self)
+ }
+}
+
+impl From<CommandTarget> for RuntimeCommandTarget {
+ fn from(target: CommandTarget) -> Self {
+ match target {
+ CommandTarget::Replica(replica) => Self::Replica(replica.as_uuid()),
+ CommandTarget::Harness(harness) => Self::Harness(harness),
}
- let body = response.text().await.unwrap_or_default();
- // 409 is the peer saying "not attached here after all" - the caller's
- // stale-view fallback keys off any error, but keep the distinction
- // visible in the report for whoever reads the logs.
- let refused = if status == StatusCode::CONFLICT {
- "the peer refused the session"
- } else {
- "the peer failed the command"
- };
- Err(HarnessError::Forward(rootcause::report!(
- "{refused}: {status}: {body}"
- )))
}
}
+
+fn current_trace_context() -> HashMap<String, String> {
+ let context = tracing::Span::current().context();
+ let mut carrier = HashMap::new();
+ opentelemetry::global::get_text_map_propagator(|propagator| {
+ propagator.inject_context(&context, &mut carrier);
+ });
+ carrier
+}
+
+fn runtime_command_span(request: &RuntimeCommandRequest) -> tracing::Span {
+ let span = tracing::info_span!(
+ "redis.process",
+ otel.kind = "consumer",
+ messaging.system = "redis",
+ messaging.operation.name = "process",
+ messaging.operation.type = "process",
+ messaging.message.id = %request.request_id,
+ agent.session.id = %request.session,
+ );
+ let parent = opentelemetry::global::get_text_map_propagator(|propagator| {
+ propagator.extract(&request.trace_context)
+ });
+ let _ = span.set_parent(parent);
+ span
+}
+
+fn forward_error(operation: &'static str, error: impl std::fmt::Display) -> HarnessError {
+ HarnessError::Forward(rootcause::report!("{operation}: {error}").into_dynamic())
+}
diff --git a/crates/agent_harness/src/outbound/runtime_registry.rs b/crates/agent_harness/src/outbound/runtime_registry.rs
index 2df9a1e97f..e88bc6e106 100644
--- a/crates/agent_harness/src/outbound/runtime_registry.rs
+++ b/crates/agent_harness/src/outbound/runtime_registry.rs
@@ -183,4 +183,8 @@ where
async fn bound_harness(&self, bot: BotId) -> anyhow::Result<Option<HarnessId>> {
self.bindings.harness_for(bot).await
}
+
+ fn is_connected(&self, harness: HarnessId) -> bool {
+ self.registry.is_connected(harness)
+ }
}
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index ca5db7ccb7..6e3094e50b 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -351,6 +351,7 @@ services:
- /var/run/docker.sock:/var/run/docker.sock
depends_on:
- postgres
+ - redis
- kafka
- connection_gateway
- lexical_service
diff --git a/infra/stacks/agent-harness-service/Pulumi.dev.yaml b/infra/stacks/agent-harness-service/Pulumi.dev.yaml
index 9188c41f0e..0a203c9756 100644
--- a/infra/stacks/agent-harness-service/Pulumi.dev.yaml
+++ b/infra/stacks/agent-harness-service/Pulumi.dev.yaml
@@ -3,3 +3,4 @@ config:
aws:region: us-east-1
agent-harness-service:github_sync_app_pem: github-sync-app-pem-dev
agent-harness-service:macro_api_token_private_secret_key: macro-api-token-private-key-dev
+ agent-harness-service:macro_cache_secret_key: macro-cache-dev
diff --git a/infra/stacks/agent-harness-service/Pulumi.prod.yaml b/infra/stacks/agent-harness-service/Pulumi.prod.yaml
index d692238d1b..acf0d2e5fa 100644
--- a/infra/stacks/agent-harness-service/Pulumi.prod.yaml
+++ b/infra/stacks/agent-harness-service/Pulumi.prod.yaml
@@ -3,3 +3,4 @@ config:
aws:region: us-east-1
agent-harness-service:github_sync_app_pem: github-sync-app-pem-prod
agent-harness-service:macro_api_token_private_secret_key: macro-api-token-private-key-prod
+ agent-harness-service:macro_cache_secret_key: macro-cache-prod
diff --git a/infra/stacks/agent-harness-service/agent_harness_service.ts b/infra/stacks/agent-harness-service/agent_harness_service.ts
index b39f2ee6da..2d20f44347 100644
--- a/infra/stacks/agent-harness-service/agent_harness_service.ts
+++ b/infra/stacks/agent-harness-service/agent_harness_service.ts
@@ -64,24 +64,12 @@ type Args = {
/**
* The agent harness service. Replicated in every environment: each replica
- * claims the sessions whose live actors it holds through a fenced Postgres
- * lease, and a command landing on the wrong replica is forwarded task-to-task
- * over the private network (`POST /internal/agent-sessions/{id}/command`,
- * internal-key authenticated) - which is why the service security group
- * allows itself ingress on the service port. The Kafka consumer group splits
- * partitions across live tasks; the lease plus forwarding is what makes that
- * split correct.
+ * claims the sessions whose live actors it holds through Postgres ownership,
+ * and commands are broadcast through the shared Redis deployment so the
+ * responsible replica can execute them. The Kafka consumer group splits
+ * partitions across live tasks; ownership plus Redis routing is what makes
+ * that split correct.
*
- * Deploys roll (min healthy 100%, max 200%): the outgoing tasks keep serving
- * until their replacements pass health checks, so the control API and egress
- * proxy stay up instead of blacking out for the old tasks' sandbox cleanup.
- * A deploy therefore doubles the replica count for its duration rather than
- * introducing coexistence - replicas coexist all the time, and the lease is
- * what makes both the steady state and the overlap correct. A command routed
- * to a new task for a session whose live actor is still on an old one
- * self-heals through resume. Live sessions already restart across every
- * deploy (`shutdown_all` stops each sandbox), so the overlap narrows the
- * blast radius rather than widening it.
*/
export class AgentHarnessService extends pulumi.ComponentResource {
public role: aws.iam.Role;
@@ -344,9 +332,6 @@ export class AgentHarnessService extends pulumi.ComponentResource {
enable: true,
rollback: true,
},
- // Rolling replace: the old task serves until the new one is healthy
- // (see the class doc comment for the overlap semantics). A failed
- // deploy rolls back with the old task still up rather than at zero.
deploymentMinimumHealthyPercent: 100,
deploymentMaximumPercent: 200,
// Every environment runs two, so dev stays prod-shaped: forwarding is
@@ -535,24 +520,6 @@ export class AgentHarnessService extends pulumi.ComponentResource {
{ parent: this }
);
- // Replica-to-replica command forwarding: a task that consumed a command
- // for a session another replica manages POSTs it directly to that
- // replica's private address, bypassing the load balancer.
- new aws.vpc.SecurityGroupIngressRule(
- `${BASE_NAME}-peer-in`,
- {
- securityGroupId: serviceSg.id,
- description:
- 'Allow harness replicas to forward session commands to each other',
- referencedSecurityGroupId: serviceSg.id,
- fromPort: serviceContainerPort,
- toPort: serviceContainerPort,
- ipProtocol: 'tcp',
- tags: this.tags,
- },
- { parent: this }
- );
-
new aws.vpc.SecurityGroupEgressRule(
`${BASE_NAME}-service-out`,
{
diff --git a/infra/stacks/agent-harness-service/index.ts b/infra/stacks/agent-harness-service/index.ts
index 205802fe07..0bc17d7cb1 100644
--- a/infra/stacks/agent-harness-service/index.ts
+++ b/infra/stacks/agent-harness-service/index.ts
@@ -45,6 +45,12 @@ const macroApiTokenPrivateKeyArn = aws.secretsmanager
const MACRO_API_TOKENS = getMacroApiToken();
+const MACRO_CACHE = aws.secretsmanager
+ .getSecretVersionOutput({
+ secretId: config.require('macro_cache_secret_key'),
+ })
+ .apply((secret) => secret.secretString);
+
// ── AI tools infra ───────────────────────────────────────────────────────────
const aiTools =
@@ -109,6 +115,10 @@ const service = new AgentHarnessService(`agent-harness-service-${stack}`, {
name: 'ENVIRONMENT',
value: stack,
},
+ {
+ name: 'REDIS_URI',
+ value: pulumi.interpolate`redis://${MACRO_CACHE}`,
+ },
// Datadog
{
name: 'DD_SERVICE',
diff --git a/services/agent_harness_service/Cargo.toml b/services/agent_harness_service/Cargo.toml
index aa92aa17cf..d5b1151f11 100644
--- a/services/agent_harness_service/Cargo.toml
+++ b/services/agent_harness_service/Cargo.toml
@@ -32,6 +32,7 @@ cursor_api_key = { path = "../../crates/cursor_api_key" }
cursor_cloud_agents = { path = "../../crates/cursor_cloud_agents" }
database_env_vars = { path = "../../crates/database_env_vars" }
entity_access = { path = "../../crates/entity_access" }
+futures = { workspace = true }
github = { path = "../../crates/github", default-features = false, features = [
"outbound",
"ports",
@@ -63,8 +64,7 @@ macro_authorization = { path = "../../crates/macro_authorization", default-featu
] }
macro_cors = { path = "../../crates/macro_cors" }
secretsmanager_client = { path = "../../crates/secretsmanager_client" }
-reqwest = { workspace = true }
-serde_json = { workspace = true }
+redis = { workspace = true, features = ["aio", "tokio-comp"] }
utoipa = { workspace = true }
utoipa-swagger-ui = { workspace = true }
macro_config = { path = "../../crates/macro_config" }
@@ -73,8 +73,10 @@ macro_env = { path = "../../crates/macro_env" }
macro_env_var = { path = "../../crates/macro_env_var" }
macro_uuid = { path = "../../crates/macro_uuid" }
serde = { workspace = true }
+serde_json = { workspace = true }
sqlx = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] }
+tokio-retry = { workspace = true }
tokio-util = { workspace = true, features = ["rt"] }
tower = { workspace = true }
tracing = { workspace = true }
diff --git a/services/agent_harness_service/src/api.rs b/services/agent_harness_service/src/api.rs
index 932989baea..4b256831c1 100644
--- a/services/agent_harness_service/src/api.rs
+++ b/services/agent_harness_service/src/api.rs
@@ -7,8 +7,6 @@ use std::time::Duration;
use agent_egress::domain::service::EgressService;
use agent_egress::inbound::axum_router::{EgressRouterState, egress_router};
-use agent_harness::domain::service::ForwardedCommands;
-use agent_harness::inbound::forward::{ForwardGatewayState, forward_router};
use agent_harness::inbound::runtime_gateway::{RuntimeGatewayState, runtime_gateway_router};
use agent_session::domain::ports::{
AgentSessionNotificationRecipient, BotDirectory, SessionOpener,
@@ -21,6 +19,8 @@ use agent_session::inbound::axum_router::{
};
use anyhow::Context;
use axum::Router;
+use axum::extract::State;
+use axum::http::StatusCode;
use axum::routing::get;
use entity_access::domain::ports::EntityAccessService;
use macro_authorization::MacroAuthorizationService;
@@ -43,8 +43,8 @@ fn mount_at_root_and_prefix(inner: Router) -> Router {
.nest(GATEWAY_PATH_PREFIX, inner)
}
-fn health_router() -> Router {
- Router::new().route("/health", get(health))
+fn health_router(ready: tokio::sync::watch::Receiver<bool>) -> Router {
+ Router::new().route("/health", get(health).with_state(ready))
}
/// Serve the sandbox-facing egress proxy on its own listener.
@@ -74,12 +74,12 @@ where
}
/// Build the router and serve it until the process is asked to stop.
-pub async fn setup_and_serve<T, R, Opener, Bots, Access, Auth, Harness>(
+pub async fn setup_and_serve<T, R, Opener, Bots, Access, Auth>(
read_state: AgentSessionRouterState<T, Access, Auth>,
control_state: AgentSessionControlState<R, Access, Auth>,
create_state: CreateSessionState<Opener, Bots, Auth>,
gateway_state: RuntimeGatewayState<Auth>,
- forward_state: ForwardGatewayState<Harness, Auth>,
+ runtime_commands_ready: tokio::sync::watch::Receiver<bool>,
port: u16,
shutdown: impl Future<Output = ()> + Send + 'static,
) -> anyhow::Result<()>
@@ -90,18 +90,11 @@ where
Bots: BotDirectory,
Access: EntityAccessService,
Auth: MacroAuthorizationService,
- Harness: ForwardedCommands,
{
- let inner = api_router(
- read_state,
- control_state,
- create_state,
- gateway_state,
- forward_state,
- )
- .layer(MacroRequestIdAndTracingLayer::new(Duration::from_millis(200)).into_inner())
- .merge(health_router())
- .layer(macro_cors::cors_layer());
+ let inner = api_router(read_state, control_state, create_state, gateway_state)
+ .layer(MacroRequestIdAndTracingLayer::new(Duration::from_millis(200)).into_inner())
+ .merge(health_router(runtime_commands_ready))
+ .layer(macro_cors::cors_layer());
let app = mount_at_root_and_prefix(inner)
.merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()))
.merge(SwaggerUi::new("/agent-harness/docs").url(
@@ -121,12 +114,11 @@ where
.context("agent harness service http failed")
}
-fn api_router<T, R, Opener, Bots, Access, Auth, Harness>(
+fn api_router<T, R, Opener, Bots, Access, Auth>(
read_state: AgentSessionRouterState<T, Access, Auth>,
control_state: AgentSessionControlState<R, Access, Auth>,
create_state: CreateSessionState<Opener, Bots, Auth>,
gateway_state: RuntimeGatewayState<Auth>,
- forward_state: ForwardGatewayState<Harness, Auth>,
) -> Router
where
T: AgentSessionService,
@@ -135,7 +127,6 @@ where
Bots: BotDirectory,
Access: EntityAccessService,
Auth: MacroAuthorizationService,
- Harness: ForwardedCommands,
{
let agent_sessions = agent_session_read_router(read_state.clone())
.merge(agent_session_control_router(control_state))
@@ -144,13 +135,12 @@ where
.nest("/agent-sessions", agent_sessions)
.merge(agent_sandbox_size_router(read_state))
.nest("/runtime", runtime_gateway_router(gateway_state))
- // Replica-to-replica command forwarding. Internal-key authenticated,
- // and reached over task-to-task networking rather than the load
- // balancer; mounting it on the public listener is fine because the
- // extractor refuses anything without the deployment's internal key.
- .nest("/internal", forward_router(forward_state))
}
-async fn health() -> &'static str {
- "ok"
+async fn health(State(ready): State<tokio::sync::watch::Receiver<bool>>) -> StatusCode {
+ if *ready.borrow() {
+ StatusCode::OK
+ } else {
+ StatusCode::SERVICE_UNAVAILABLE
+ }
}
diff --git a/services/agent_harness_service/src/api/test.rs b/services/agent_harness_service/src/api/test.rs
index cd34517c60..c97928b719 100644
--- a/services/agent_harness_service/src/api/test.rs
+++ b/services/agent_harness_service/src/api/test.rs
@@ -8,7 +8,7 @@ use tower::ServiceExt;
#[tokio::test]
async fn health_is_reachable_at_root_and_gateway_prefix() {
for path in ["/health", "/agent-harness/health"] {
- let response = mount_at_root_and_prefix(health_router())
+ let response = mount_at_root_and_prefix(health_router(tokio::sync::watch::channel(true).1))
.oneshot(
Request::builder()
.uri(path)
@@ -25,7 +25,7 @@ async fn health_is_reachable_at_root_and_gateway_prefix() {
#[tokio::test]
async fn unprefixed_unknown_path_is_not_rewritten_onto_the_prefix() {
- let response = mount_at_root_and_prefix(health_router())
+ let response = mount_at_root_and_prefix(health_router(tokio::sync::watch::channel(true).1))
.oneshot(
Request::builder()
.uri("/missing")
@@ -38,3 +38,19 @@ async fn unprefixed_unknown_path_is_not_rewritten_onto_the_prefix() {
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
+
+#[tokio::test]
+async fn health_fails_while_the_command_bus_is_disconnected() {
+ let response = health_router(tokio::sync::watch::channel(false).1)
+ .oneshot(
+ Request::builder()
+ .uri("/health")
+ .method("GET")
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+}
diff --git a/services/agent_harness_service/src/config.rs b/services/agent_harness_service/src/config.rs
index 7079b02f84..36c7f63513 100644
--- a/services/agent_harness_service/src/config.rs
+++ b/services/agent_harness_service/src/config.rs
@@ -2,7 +2,7 @@
//! `macro_config` pattern.
use anyhow::Context;
-use database_env_vars::DatabaseUrl;
+use database_env_vars::{DatabaseUrl, RedisUri};
pub use macro_env::Environment;
use macro_uuid::Uuid;
@@ -51,6 +51,8 @@ pub struct Config {
pub kafka_brokers: KafkaBrokers,
/// MacroDB connection string; `agent_sessions` lives here.
pub database_url: DatabaseUrl,
+ /// Shared Redis used for cross-replica command forwarding.
+ pub redis_uri: RedisUri,
/// Base URL of the Daytona REST API.
#[macro_config_default(String::from("https://app.daytona.io/api"))]
pub daytona_api_url: String,
diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rs
index 597a00da70..228ee3312d 100644
--- a/services/agent_harness_service/src/main.rs
+++ b/services/agent_harness_service/src/main.rs
@@ -13,6 +13,7 @@ mod bots_directory;
mod config;
mod containers;
mod harness_bindings;
+mod runtime_commands;
mod trigger;
use std::{future::Future, pin::Pin, sync::Arc};
@@ -32,7 +33,6 @@ use agent_harness::domain::service::AgentHarnessService;
use agent_harness::domain::trigger_router::{
RoutedTrigger, agent_trigger_bot_id, route_agent_trigger,
};
-use agent_harness::inbound::forward::ForwardGatewayState;
use agent_harness::inbound::runtime_gateway::RuntimeGatewayState;
use agent_harness::outbound::agent_prompt_composer::LexicalAgentPromptComposer;
use agent_harness::outbound::channel_announcer::ChannelAnnouncer;
@@ -44,7 +44,7 @@ use agent_harness::outbound::daytona::{
DaytonaContainerManager, DaytonaSettings, Snapshot,
};
use agent_harness::outbound::egress::EgressProvisioner;
-use agent_harness::outbound::forward::HttpCommandForwarder;
+use agent_harness::outbound::forward::RedisCommandForwarder;
use agent_harness::outbound::local::{LocalContainerManager, LocalSettings};
use agent_harness::outbound::routing::RoutedContainerManager;
use agent_harness::outbound::runtime_registry::{HarnessKeyedConnections, RuntimeRegistry};
@@ -52,7 +52,7 @@ use agent_inmem::outbound::log_frames::LogFrameSource;
use agent_inmem::outbound::manager::InMemAgentManager;
use agent_inmem::outbound::rig_engine::RigTurnEngine;
use agent_runtime_directory::PgAgentRuntimeDirectory;
-use agent_session::domain::model::{ReplicaAddress, ReplicaId};
+use agent_session::domain::model::ReplicaId;
use agent_session::domain::ports::{NoOpRealtime, SessionOwnership as _};
use agent_session::domain::service::AgentSessionServiceImpl;
use agent_session::inbound::axum_router::{
@@ -102,8 +102,11 @@ use pipedream_mcp::outbound::pg_connection_repo::PgConnectionRepo;
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
use sqlx::postgres::PgPoolOptions;
+use tokio_retry::{Retry, strategy::FixedInterval};
use tracing::Instrument as _;
+use runtime_commands::consume_runtime_commands;
+
/// Consumer group owning this harness's agent-session offsets.
///
/// TODO: one group per bot deployment. Two bots sharing this name would
@@ -111,6 +114,8 @@ use tracing::Instrument as _;
/// exactly one harness deployment exists.
struct AgentHarnessConsumerGroup;
+const RUNTIME_COMMAND_CONSUMER_ATTEMPTS: usize = 5;
+
impl GroupName for AgentHarnessConsumerGroup {
const GROUP_NAME: &'static str = "agent-harness-service";
}
@@ -137,63 +142,6 @@ struct PendingHarnessWork {
description: &'static str,
}
-/// This task's own private base URL, for peers forwarding session commands.
-///
-/// On ECS the task metadata endpoint (v4) publishes the task's private IPv4;
-/// the port is the service listener peers can reach through the task-to-task
-/// security group rule. Local development has no routable address and runs
-/// one replica, so `None` is correct there; every other environment refuses
-/// to boot rather than heartbeat an unreachable identity, because a replica
-/// without an address is one whose sessions peers can see but never reach.
-async fn discover_replica_address(
- environment: Environment,
- port: u16,
-) -> anyhow::Result<Option<agent_session::domain::model::ReplicaAddress>> {
- // Explicit override first: how a local multi-replica experiment names
- // each process (`REPLICA_ADDRESS=http://127.0.0.1:<port>`), since nothing
- // off ECS can discover an address.
- if let Some(address) = macro_env_var::optional_read_env_var("REPLICA_ADDRESS")
- .ok()
- .flatten()
- {
- return Ok(Some(agent_session::domain::model::ReplicaAddress::new(
- address,
- )));
- }
- let Some(metadata_uri) = macro_env_var::optional_read_env_var("ECS_CONTAINER_METADATA_URI_V4")
- .ok()
- .flatten()
- else {
- if matches!(environment, Environment::Local) {
- return Ok(None);
- }
- anyhow::bail!(
- "ECS_CONTAINER_METADATA_URI_V4 is unset outside a local environment: \
- this replica would be unreachable for command forwarding"
- );
- };
- let task: serde_json::Value = reqwest::Client::new()
- .get(format!("{metadata_uri}/task"))
- .timeout(std::time::Duration::from_secs(5))
- .send()
- .await
- .context("failed to reach the ECS task metadata endpoint")?
- .json()
- .await
- .context("failed to parse ECS task metadata")?;
- let ip = task["Containers"]
- .as_array()
- .into_iter()
- .flatten()
- .flat_map(|container| container["Networks"].as_array().into_iter().flatten())
- .flat_map(|network| network["IPv4Addresses"].as_array().into_iter().flatten())
- .find_map(serde_json::Value::as_str)
- .context("ECS task metadata carries no private IPv4 address")?;
- Ok(Some(agent_session::domain::model::ReplicaAddress::new(
- format!("http://{ip}:{port}"),
- )))
-}
-
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let entrypoint = MacroEntrypoint::default().init();
@@ -246,18 +194,10 @@ async fn run() -> anyhow::Result<()> {
// log and pushes each frame at the channel's participants so a viewer sees
// it happen.
let session_repo = PgAgentSessionRepo::new(pool.clone());
- // One lease identity for the whole process: commands forward to an
- // address, and both attach-capable service instances below live at this
- // one, so they share the id. The address is this task's own private URL,
- // discovered from ECS metadata; a deployment that cannot learn it must
- // not scale, so discovery failing anywhere but local refuses to boot.
+ // One ownership identity for the whole process. Both attach-capable
+ // service instances below live here, so they share it.
let replica = ReplicaId::mint();
- let replica_address = discover_replica_address(config.environment, config.port).await?;
- tracing::info!(
- %replica,
- address = ?replica_address.as_ref().map(ReplicaAddress::as_str),
- "harness replica identity"
- );
+ tracing::info!(%replica, "harness replica identity");
// Bound to the harness once it exists (it is built *from* this service);
// both attach-capable service instances report turns to the same one.
let turn_observer = Arc::new(agent_session::domain::ports::LateBoundTurnObserver::new());
@@ -466,6 +406,8 @@ async fn run() -> anyhow::Result<()> {
// it and the harness takes sessions out of it. Attach/detach is mirrored
// to the harnesses table so the settings page can show connection state.
let runtimes = RuntimeRegistry::with_presence(Arc::new(PgHarnessPresence::new(pool.clone())));
+ let redis = redis::Client::open(config.redis_uri.as_ref())
+ .context("failed to create the runtime command Redis client")?;
let mut defaults = HarnessDefaults::new(SessionDefaults {
bot_id,
model: config.harness_model.clone(),
@@ -518,13 +460,50 @@ async fn run() -> anyhow::Result<()> {
prompt_context,
prompt_composer,
EgressProvisioner::new(Arc::clone(&mcp_connections), config.egress_base_url.clone()),
- HttpCommandForwarder::new(config.internal_api_key.clone())
- .map_err(|error| anyhow::anyhow!("failed to build the command forwarder: {error}"))?,
+ RedisCommandForwarder::new(redis.clone()),
defaults,
));
// Close the loop: turn ends observed by the session actors drain the
// harness's prompt queue.
turn_observer.bind(harness.clone());
+ let runtime_command_redis = redis.clone();
+ let runtime_command_harness = harness.clone();
+ let runtime_command_runtimes = Arc::clone(&runtimes);
+ let (runtime_commands_ready, mut runtime_commands_readiness) =
+ tokio::sync::watch::channel(false);
+ let runtime_commands = tokio::spawn(async move {
+ let result = Retry::start(
+ FixedInterval::new(std::time::Duration::from_secs(1))
+ .take(RUNTIME_COMMAND_CONSUMER_ATTEMPTS - 1),
+ || {
+ runtime_commands_ready.send_replace(false);
+ consume_runtime_commands(
+ runtime_command_redis.clone(),
+ replica,
+ {
+ let runtimes = Arc::clone(&runtime_command_runtimes);
+ Arc::new(move |harness| runtimes.is_connected(harness))
+ },
+ runtime_command_harness.clone(),
+ runtime_commands_ready.clone(),
+ )
+ },
+ )
+ .await;
+ if let Err(error) = result {
+ tracing::error!(
+ error = ?error,
+ "runtime command Redis consumer stopped after five attempts"
+ );
+ }
+ });
+ tokio::time::timeout(
+ std::time::Duration::from_secs(30),
+ runtime_commands_readiness.wait_for(|ready| *ready),
+ )
+ .await
+ .context("timed out subscribing to the runtime command bus")?
+ .context("runtime command bus stopped before subscribing")?;
// The complete session API is served from this process because it owns the
// live sessions. Spawned rather than awaited: the Kafka loop below owns the
@@ -567,10 +546,7 @@ async fn run() -> anyhow::Result<()> {
runtimes,
MacroAuthorizationState::new(Arc::new(authorization_service.clone())),
);
- let forward_state = ForwardGatewayState::new(
- harness.clone(),
- MacroAuthorizationState::new(Arc::new(authorization_service)),
- );
+ let http_runtime_commands_readiness = runtime_commands_readiness.clone();
let http_port = config.port;
let http = tokio::spawn(async move {
if let Err(error) = api::setup_and_serve(
@@ -578,7 +554,7 @@ async fn run() -> anyhow::Result<()> {
control_state,
create_state,
gateway_state,
- forward_state,
+ http_runtime_commands_readiness,
http_port,
shutdown_signal(),
)
@@ -594,16 +570,16 @@ async fn run() -> anyhow::Result<()> {
// release each claim eagerly in the actor teardown path; this loop is
// what covers the ungraceful ones.
let heartbeat_repo = session_repo.clone();
- let heartbeat_address = replica_address.clone();
+ let heartbeat_readiness = runtime_commands_readiness.clone();
let heartbeat = tokio::spawn(async move {
let mut ticker =
tokio::time::interval(agent_session::domain::ports::REPLICA_HEARTBEAT_INTERVAL);
loop {
ticker.tick().await;
- if let Err(error) = heartbeat_repo
- .heartbeat(replica, heartbeat_address.as_ref())
- .await
- {
+ if !*heartbeat_readiness.borrow() {
+ continue;
+ }
+ if let Err(error) = heartbeat_repo.heartbeat(replica, None).await {
tracing::warn!(error = ?error, %replica, "failed to heartbeat harness replica");
}
}
@@ -837,6 +813,7 @@ async fn run() -> anyhow::Result<()> {
trigger.abort();
egress_http.abort();
heartbeat.abort();
+ runtime_commands.abort();
let stop_failures = container_shutdown.shutdown_all().await;
if stop_failures > 0 {
tracing::error!(stop_failures, "some sandboxes failed to stop");
diff --git a/services/agent_harness_service/src/runtime_commands.rs b/services/agent_harness_service/src/runtime_commands.rs
new file mode 100644
index 0000000000..2e710ff6e7
--- /dev/null
+++ b/services/agent_harness_service/src/runtime_commands.rs
@@ -0,0 +1,93 @@
+//! Redis command-bus listener composed around the harness domain service.
+
+use std::sync::Arc;
+
+use agent_harness::domain::service::ForwardedCommands;
+use agent_harness::outbound::forward::{
+ COMMAND_CHANNEL, RuntimeCommandRequest, RuntimeCommandTarget,
+};
+use agent_session::domain::model::ReplicaId;
+use futures::StreamExt as _;
+use harness_id::HarnessId;
+use redis::AsyncCommands as _;
+use redis::{ExistenceCheck, SetExpiry, SetOptions};
+use tracing::Instrument as _;
+
+#[cfg(test)]
+mod test;
+
+pub(crate) async fn consume_runtime_commands<Harness>(
+ redis: redis::Client,
+ replica: ReplicaId,
+ connected: Arc<dyn Fn(HarnessId) -> bool + Send + Sync>,
+ harness_service: Arc<Harness>,
+ ready: tokio::sync::watch::Sender<bool>,
+) -> anyhow::Result<()>
+where
+ Harness: ForwardedCommands,
+{
+ let mut subscriber = redis.get_async_pubsub().await?;
+ subscriber.subscribe(COMMAND_CHANNEL).await?;
+ ready.send_replace(true);
+ let mut requests = subscriber.into_on_message();
+ while let Some(request) = requests.next().await {
+ let payload: String = match request.get_payload() {
+ Ok(payload) => payload,
+ Err(error) => {
+ tracing::warn!(error = ?error, "dropping malformed runtime command broadcast");
+ continue;
+ }
+ };
+ let request: RuntimeCommandRequest = match serde_json::from_str(&payload) {
+ Ok(request) => request,
+ Err(error) => {
+ tracing::warn!(error = ?error, "dropping malformed runtime command broadcast");
+ continue;
+ }
+ };
+ let redis = redis.clone();
+ let harness_service = Arc::clone(&harness_service);
+ let connected = Arc::clone(&connected);
+ let span = request.processing_span();
+ tokio::spawn(
+ async move {
+ let request_id = request.request_id();
+ let session = request.session();
+ let selected = match request.target() {
+ RuntimeCommandTarget::Replica(target) => target == replica.as_uuid(),
+ RuntimeCommandTarget::Harness(harness) if connected(harness) => {
+ let mut connection = match redis.get_multiplexed_async_connection().await {
+ Ok(connection) => connection,
+ Err(error) => {
+ tracing::error!(error = ?error, "failed to claim a harness command");
+ return;
+ }
+ };
+ connection
+ .set_options::<_, _, Option<String>>(
+ format!("agent-harness.runtime-command.claim.{request_id}"),
+ replica.as_uuid().to_string(),
+ SetOptions::default()
+ .conditional_set(ExistenceCheck::NX)
+ .with_expiration(SetExpiry::EX(120)),
+ )
+ .await
+ .is_ok_and(|result| result.is_some())
+ }
+ RuntimeCommandTarget::Harness(_) => false,
+ };
+ if selected {
+ harness_service
+ .execute_forwarded(session, request.into_command())
+ .await
+ .inspect_err(|error| {
+ tracing::error!(error = ?error, %session, "forwarded command failed");
+ })
+ .ok();
+ }
+ }
+ .instrument(span),
+ );
+ }
+ anyhow::bail!("runtime command Redis subscription ended")
+}
diff --git a/services/agent_harness_service/src/runtime_commands/test.rs b/services/agent_harness_service/src/runtime_commands/test.rs
new file mode 100644
index 0000000000..56eefb5158
--- /dev/null
+++ b/services/agent_harness_service/src/runtime_commands/test.rs
@@ -0,0 +1,214 @@
+use super::*;
+use agent_harness::domain::error::{HarnessError, Result};
+use agent_harness::domain::model::{CommandOutcome, HarnessCommand};
+use agent_harness::domain::ports::{CommandForwarder as _, CommandTarget};
+use agent_harness::outbound::forward::RedisCommandForwarder;
+use agent_session::domain::model::AgentSessionId;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+struct RecordingHarness {
+ calls: AtomicUsize,
+ fails: bool,
+}
+
+impl ForwardedCommands for RecordingHarness {
+ async fn execute_forwarded(
+ &self,
+ session_id: AgentSessionId,
+ _command: HarnessCommand,
+ ) -> Result<CommandOutcome> {
+ self.calls.fetch_add(1, Ordering::SeqCst);
+ if self.fails {
+ return Err(HarnessError::Disconnected(session_id));
+ }
+ Ok(CommandOutcome::Completed)
+ }
+}
+
+fn redis_client() -> redis::Client {
+ let url = macro_env_var::optional_read_env_var("REDIS_URI")
+ .ok()
+ .flatten()
+ .unwrap_or_else(|| "redis://127.0.0.1:6379".to_owned());
+ redis::Client::open(url).expect("valid Redis URL")
+}
+
+async fn ready(readiness: &mut tokio::sync::watch::Receiver<bool>) {
+ tokio::time::timeout(
+ std::time::Duration::from_secs(2),
+ readiness.wait_for(|ready| *ready),
+ )
+ .await
+ .expect("consumer becomes ready")
+ .expect("readiness channel remains open");
+}
+
+async fn wait_for_calls(harness: &RecordingHarness, expected: usize) {
+ tokio::time::timeout(std::time::Duration::from_secs(2), async {
+ while harness.calls.load(Ordering::SeqCst) != expected {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("command is processed");
+}
+
+fn harness(fails: bool) -> Arc<RecordingHarness> {
+ Arc::new(RecordingHarness {
+ calls: AtomicUsize::new(0),
+ fails,
+ })
+}
+
+fn harness_id() -> HarnessId {
+ HarnessId::new_from_uuid(macro_uuid::generate_uuid_v7())
+}
+
+#[tokio::test]
+async fn a_command_executes_on_exactly_one_responsible_replica() {
+ let redis = redis_client();
+ let origin = ReplicaId::mint();
+ let peer = ReplicaId::mint();
+ let origin_harness = harness(false);
+ let peer_harness = harness(false);
+ let (origin_ready, mut origin_readiness) = tokio::sync::watch::channel(false);
+ let origin_consumer = tokio::spawn(consume_runtime_commands(
+ redis.clone(),
+ origin,
+ Arc::new(|_| false),
+ Arc::clone(&origin_harness),
+ origin_ready,
+ ));
+ let (peer_ready, mut peer_readiness) = tokio::sync::watch::channel(false);
+ let peer_consumer = tokio::spawn(consume_runtime_commands(
+ redis.clone(),
+ peer,
+ Arc::new(|_| false),
+ Arc::clone(&peer_harness),
+ peer_ready,
+ ));
+ ready(&mut origin_readiness).await;
+ ready(&mut peer_readiness).await;
+
+ let outcome = RedisCommandForwarder::new(redis)
+ .forward(
+ AgentSessionId::TEST_A,
+ HarnessCommand::Delete,
+ CommandTarget::Replica(peer),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(outcome, CommandOutcome::Completed);
+ wait_for_calls(&peer_harness, 1).await;
+ assert_eq!(origin_harness.calls.load(Ordering::SeqCst), 0);
+ origin_consumer.abort();
+ peer_consumer.abort();
+}
+
+#[tokio::test]
+async fn a_responsible_replicas_error_does_not_fail_the_publish() {
+ let redis = redis_client();
+ let replica = ReplicaId::mint();
+ let harness = harness(true);
+ let (ready_tx, mut readiness) = tokio::sync::watch::channel(false);
+ let consumer = tokio::spawn(consume_runtime_commands(
+ redis.clone(),
+ replica,
+ Arc::new(|_| false),
+ Arc::clone(&harness),
+ ready_tx,
+ ));
+ ready(&mut readiness).await;
+
+ let outcome = RedisCommandForwarder::new(redis)
+ .forward(
+ AgentSessionId::TEST_A,
+ HarnessCommand::Delete,
+ CommandTarget::Replica(replica),
+ )
+ .await
+ .expect("the command publishes");
+
+ assert_eq!(outcome, CommandOutcome::Completed);
+ wait_for_calls(&harness, 1).await;
+ consumer.abort();
+}
+
+#[tokio::test]
+async fn publishing_without_an_owner_still_succeeds() {
+ let redis = redis_client();
+ let harness = harness(false);
+ let (ready_tx, mut readiness) = tokio::sync::watch::channel(false);
+ let consumer = tokio::spawn(consume_runtime_commands(
+ redis.clone(),
+ ReplicaId::mint(),
+ Arc::new(|_| false),
+ Arc::clone(&harness),
+ ready_tx,
+ ));
+ ready(&mut readiness).await;
+
+ let target = harness_id();
+ let outcome = RedisCommandForwarder::new(redis)
+ .forward(
+ AgentSessionId::TEST_A,
+ HarnessCommand::Delete,
+ CommandTarget::Harness(target),
+ )
+ .await
+ .expect("the command publishes");
+
+ assert_eq!(outcome, CommandOutcome::Completed);
+ assert_eq!(harness.calls.load(Ordering::SeqCst), 0);
+ consumer.abort();
+}
+
+#[tokio::test]
+async fn overlapping_harness_connections_execute_once() {
+ let redis = redis_client();
+ let first = harness(false);
+ let second = harness(false);
+ let target = harness_id();
+ let (first_ready, mut first_readiness) = tokio::sync::watch::channel(false);
+ let first_consumer = tokio::spawn(consume_runtime_commands(
+ redis.clone(),
+ ReplicaId::mint(),
+ Arc::new(move |harness| harness == target),
+ Arc::clone(&first),
+ first_ready,
+ ));
+ let (second_ready, mut second_readiness) = tokio::sync::watch::channel(false);
+ let second_consumer = tokio::spawn(consume_runtime_commands(
+ redis.clone(),
+ ReplicaId::mint(),
+ Arc::new(move |harness| harness == target),
+ Arc::clone(&second),
+ second_ready,
+ ));
+ ready(&mut first_readiness).await;
+ ready(&mut second_readiness).await;
+
+ RedisCommandForwarder::new(redis)
+ .forward(
+ AgentSessionId::TEST_A,
+ HarnessCommand::Delete,
+ CommandTarget::Harness(target),
+ )
+ .await
+ .unwrap();
+
+ tokio::time::timeout(std::time::Duration::from_secs(2), async {
+ while first.calls.load(Ordering::SeqCst) + second.calls.load(Ordering::SeqCst) != 1 {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("command is processed");
+ assert_eq!(
+ first.calls.load(Ordering::SeqCst) + second.calls.load(Ordering::SeqCst),
+ 1
+ );
+ first_consumer.abort();
+ second_consumer.abort();
+}