Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(agent_harness): horizontal harness — rolling deploys, fenced session lease, command forwarding (#6032)
Wolf Mermelstein7d07f6d4f03d
diff --git a/.sqlx/query-18fd730ea7a2734ac0d674edec980025a77971dac52498a2adb9470e6dc73787.json b/.sqlx/query-18fd730ea7a2734ac0d674edec980025a77971dac52498a2adb9470e6dc73787.jsonnew file mode 100644index 0000000000..34ca00b9bf--- /dev/null+++ b/.sqlx/query-18fd730ea7a2734ac0d674edec980025a77971dac52498a2adb9470e6dc73787.json@@ -0,0 +1,28 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content)\n SELECT $1, $2, $3, $4, $5\n WHERE EXISTS (\n SELECT 1 FROM agent_session\n WHERE id = $2 AND manager_replica_id = $6 AND manager_fence = $7\n )\n RETURNING created_at\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "created_at",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid",+ "Text",+ "Text",+ "Jsonb",+ "Uuid",+ "Int8"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "18fd730ea7a2734ac0d674edec980025a77971dac52498a2adb9470e6dc73787"+}diff --git a/.sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.json b/.sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.jsonnew file mode 100644index 0000000000..6dc5347586--- /dev/null+++ b/.sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.json@@ -0,0 +1,24 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH replica AS (\n INSERT INTO harness_replica (id, last_heartbeat_at)\n VALUES ($2, now())\n ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()\n )\n UPDATE agent_session\n SET manager_replica_id = $2,\n manager_fence = manager_fence + 1,\n modified_at = now()\n WHERE id = $1\n AND (\n manager_replica_id IS NULL\n OR manager_replica_id = $2\n OR NOT EXISTS (\n SELECT 1 FROM harness_replica live\n WHERE live.id = agent_session.manager_replica_id\n AND live.last_heartbeat_at > now() - make_interval(secs => $3)\n )\n )\n RETURNING manager_fence\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "manager_fence",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid",+ "Float8"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd"+}diff --git a/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.json b/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.jsonnew file mode 100644index 0000000000..e0fc201db3--- /dev/null+++ b/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.json@@ -0,0 +1,29 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT live.id AS replica_id, live.address\n FROM agent_session\n JOIN harness_replica live ON live.id = agent_session.manager_replica_id\n WHERE agent_session.id = $1\n AND live.last_heartbeat_at > now() - make_interval(secs => $2)\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "replica_id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "address",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Float8"+ ]+ },+ "nullable": [+ false,+ true+ ]+ },+ "hash": "261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52"+}diff --git a/.sqlx/query-521b591873e9fdbaa3310380723aafad9da0742692aec0122c5c922804c5f795.json b/.sqlx/query-521b591873e9fdbaa3310380723aafad9da0742692aec0122c5c922804c5f795.jsonnew file mode 100644index 0000000000..3e2d2c8469--- /dev/null+++ b/.sqlx/query-521b591873e9fdbaa3310380723aafad9da0742692aec0122c5c922804c5f795.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE agent_session\n SET manager_replica_id = NULL,\n modified_at = now()\n WHERE id = $1 AND manager_replica_id = $2 AND manager_fence = $3\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid",+ "Int8"+ ]+ },+ "nullable": []+ },+ "hash": "521b591873e9fdbaa3310380723aafad9da0742692aec0122c5c922804c5f795"+}diff --git a/.sqlx/query-60620ad0a9e7fd8a9c8c61c331636e18332c96647f0b87bc29067d41104dd057.json b/.sqlx/query-60620ad0a9e7fd8a9c8c61c331636e18332c96647f0b87bc29067d41104dd057.jsonnew file mode 100644index 0000000000..f2674f6222--- /dev/null+++ b/.sqlx/query-60620ad0a9e7fd8a9c8c61c331636e18332c96647f0b87bc29067d41104dd057.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO harness_replica (id, last_heartbeat_at, address)\n VALUES ($1, now(), $2)\n ON CONFLICT (id) DO UPDATE\n SET last_heartbeat_at = now(),\n address = COALESCE(EXCLUDED.address, harness_replica.address)\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "60620ad0a9e7fd8a9c8c61c331636e18332c96647f0b87bc29067d41104dd057"+}diff --git a/.sqlx/query-8b9476b82011c40253d7dd36078e92e9fc4e0e295c9031b76879a8b472bb2aa5.json b/.sqlx/query-8b9476b82011c40253d7dd36078e92e9fc4e0e295c9031b76879a8b472bb2aa5.jsonnew file mode 100644index 0000000000..248f9aaf6c--- /dev/null+++ b/.sqlx/query-8b9476b82011c40253d7dd36078e92e9fc4e0e295c9031b76879a8b472bb2aa5.json@@ -0,0 +1,12 @@+{+ "db_name": "PostgreSQL",+ "query": "DELETE FROM harness_replica WHERE last_heartbeat_at < now() - interval '7 days'",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": []+ },+ "nullable": []+ },+ "hash": "8b9476b82011c40253d7dd36078e92e9fc4e0e295c9031b76879a8b472bb2aa5"+}diff --git a/.sqlx/query-a6d600a2285024d53f613e78fac8e24c64a44430b34b3f2f82edd6426ec26751.json b/.sqlx/query-a6d600a2285024d53f613e78fac8e24c64a44430b34b3f2f82edd6426ec26751.jsonnew file mode 100644index 0000000000..9845f3c170--- /dev/null+++ b/.sqlx/query-a6d600a2285024d53f613e78fac8e24c64a44430b34b3f2f82edd6426ec26751.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT manager_replica_id FROM agent_session WHERE id = $1",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "manager_replica_id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ true+ ]+ },+ "hash": "a6d600a2285024d53f613e78fac8e24c64a44430b34b3f2f82edd6426ec26751"+}diff --git a/Cargo.lock b/Cargo.lockindex 066eb86354..4088db34ff 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -324,6 +324,7 @@ dependencies = ["notification","pipedream_mcp","rdkafka",+ "reqwest 0.13.4","secretsmanager_client","serde","serde_json",diff --git a/crates/agent_harness/src/domain/error.rs b/crates/agent_harness/src/domain/error.rsindex 7cc90717a3..73fc52d707 100644--- a/crates/agent_harness/src/domain/error.rs+++ b/crates/agent_harness/src/domain/error.rs@@ -63,4 +63,12 @@ pub enum HarnessError {/// A prompt could not be composed for the agent runtime.#[error("failed to compose agent prompt: {0}")]PromptComposition(rootcause::Report),+ /// A live peer manages the session but published no address to forward+ /// to. Transient by construction: only a replica from before addresses+ /// existed, or one whose first heartbeat has not landed, has no address.+ #[error("agent session {0} is managed by a live replica with no forwarding address")]+ ManagerUnreachable(AgentSessionId),+ /// Forwarding a command to the session's managing replica failed.+ #[error("failed to forward an agent session command: {0}")]+ Forward(rootcause::Report),}diff --git a/crates/agent_harness/src/domain/model.rs b/crates/agent_harness/src/domain/model.rsindex b8e9304d22..e1e8fc9892 100644--- a/crates/agent_harness/src/domain/model.rs+++ b/crates/agent_harness/src/domain/model.rs@@ -9,7 +9,7 @@ use bot_id::BotId;use macro_user_id::user_id::MacroUserIdStr;use macro_uuid::Uuid;/// Where a mention happened.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]pub struct MentionOrigin {/// Channel the mentioning message was posted in.pub channel_id: Uuid,@@ -30,7 +30,7 @@ pub struct MentionOrigin {/// [`agent_session::domain::ports::SessionOpener`] instead: they/// need no provisioning, no announcement, and no first prompt, so they are/// a plain create rather than a harness command.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]pub struct OpenSession {/// The bot that was mentioned.pub bot_id: BotId,@@ -105,7 +105,7 @@ pub(crate) use agent_egress::domain::model::is_macro_staff;/// Where a prompt came from, when it came from somewhere the session should/// answer back into.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]pub struct AnnounceOrigin {/// Channel the prompt was posted in.pub channel_id: Uuid,@@ -125,7 +125,7 @@ pub struct PriorChannelMessage {}/// Do something in a session that already exists.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]pub struct DeliverAction {/// The id the action carries onto the wire, minted when it was accepted./// A reconnect-and-retry resends under the same id.@@ -151,7 +151,7 @@ pub struct DeliverAction {/// commands - whether to reconnect a dead session, whether to announce - are/// properties of the action and its origin, not of the request that carried/// it.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]pub enum HarnessCommand {/// Open a new session.Open(OpenSession),diff --git a/crates/agent_harness/src/domain/ports.rs b/crates/agent_harness/src/domain/ports.rsindex 864f299fda..d2ba3f3afe 100644--- a/crates/agent_harness/src/domain/ports.rs+++ b/crates/agent_harness/src/domain/ports.rs@@ -1,18 +1,56 @@//! Outbound capabilities required by the harness domain.use agent_session::domain::connection::RuntimeAttachment;-use agent_session::domain::model::{AgentSessionId, SandboxSize};+use agent_session::domain::model::{AgentSessionId, ReplicaAddress, SandboxSize};use agent_session::domain::ports::AgentConnector;use bot_id::BotId;use macro_user_id::user_id::MacroUserIdStr;-use super::error::Result;+use super::error::{HarnessError, Result};use super::model::{- PriorChannelMessage, ProvisionedEgress, SandboxEgress, SessionAnnouncement, SpawnContainer,+ HarnessCommand, PriorChannelMessage, ProvisionedEgress, SandboxEgress, SessionAnnouncement,+ SpawnContainer,};use super::sandbox::SandboxResizeEffect;+/// Delivers a session's command to the replica that manages its live actor.+///+/// The receiving side executes without re-resolving management (a forward is+/// single-hop by contract, so two replicas with momentarily different views+/// cannot bounce a command between each other). Success means the peer ran+/// the command to completion - the response is the acknowledgment - so a+/// caller that awaited a forward has the same guarantee as one that executed+/// locally.+pub trait CommandForwarder: Send + Sync + 'static {+ /// Run `command` for `session` on the replica at `target`.+ fn forward(+ &self,+ target: &ReplicaAddress,+ session: AgentSessionId,+ command: HarnessCommand,+ ) -> impl Future<Output = Result<()>> + Send;+}++/// A forwarder for deployments with exactly one replica, where a live peer+/// cannot exist: being asked to forward is itself the error, loudly, rather+/// than a silent local fallback that would mask a mis-wiring.+#[derive(Debug, Clone, Copy, Default)]+pub struct NoPeers;++impl CommandForwarder for NoPeers {+ async fn forward(+ &self,+ target: &ReplicaAddress,+ session: AgentSessionId,+ _command: HarnessCommand,+ ) -> Result<()> {+ Err(HarnessError::Forward(rootcause::report!(+ "this deployment has no command forwarding, yet {target} manages session {session}"+ )))+ }+}+#[cfg(test)]mod test;diff --git a/crates/agent_harness/src/domain/service.rs b/crates/agent_harness/src/domain/service.rsindex 77a613b121..61cc2c0275 100644--- a/crates/agent_harness/src/domain/service.rs+++ b/crates/agent_harness/src/domain/service.rs@@ -6,6 +6,7 @@ use std::sync::Arc;use agent_runtime_protocol::domain::action::{AgentAction, AgentActionId};use agent_session::domain::connection::RuntimeAttachment;use agent_session::domain::error::AgentSessionError;+use agent_session::domain::model::SessionManagement;use agent_session::domain::model::{AgentSession, AgentSessionId, AuthorKind, CreateAgentSessionParams, MessageId, SandboxSize,};@@ -25,8 +26,8 @@ use crate::domain::model::{OpenSession, SessionAnnouncement, SpawnContainer, is_macro_staff,};use crate::domain::ports::{- AgentPromptComposer, ChannelPromptContext, ContainerManager, RuntimeConnections,- SandboxEgressProvisioner, SessionAnnouncer,+ AgentPromptComposer, ChannelPromptContext, CommandForwarder, ContainerManager,+ RuntimeConnections, SandboxEgressProvisioner, SessionAnnouncer,};use crate::domain::sandbox::SandboxResizeEffect;@@ -38,6 +39,37 @@ struct QueuedCommand {/// The caller's span, carried across the queue so the work the worker does/// on its own task still hangs off whatever triggered it.span: tracing::Span,+ /// Whether the worker resolves the session's managing replica before+ /// executing. Commands admitted at an ingress route; a command received+ /// *as* a forward executes here unconditionally, which is what makes+ /// forwarding single-hop - two replicas with momentarily different lease+ /// views cannot bounce a command between each other.+ route: bool,+}++/// [`CommandForwarder`], object-safe.+///+/// Held erased inside the service so forwarding does not become an eighth+/// type parameter on every impl block; the public port keeps its natural+/// `impl Future` shape and this shim boxes at the one internal call site.+trait ErasedForwarder: Send + Sync + 'static {+ fn forward<'a>(+ &'a self,+ target: &'a agent_session::domain::model::ReplicaAddress,+ session: AgentSessionId,+ command: HarnessCommand,+ ) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>>;+}++impl<F: CommandForwarder> ErasedForwarder for F {+ fn forward<'a>(+ &'a self,+ target: &'a agent_session::domain::model::ReplicaAddress,+ session: AgentSessionId,+ command: HarnessCommand,+ ) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {+ Box::pin(CommandForwarder::forward(self, target, session, command))+ }}struct AgentHarnessInner<@@ -56,6 +88,7 @@ struct AgentHarnessInner<prompt_context: PromptContext,prompt_composer: PromptComposer,egress: Egress,+ forwarder: Box<dyn ErasedForwarder>,defaults: HarnessDefaults,}@@ -115,6 +148,7 @@ whereprompt_context: PromptContext,prompt_composer: PromptComposer,egress: Egress,+ forwarder: impl CommandForwarder,defaults: impl Into<HarnessDefaults>,) -> Self {Self {@@ -126,6 +160,7 @@ whereprompt_context,prompt_composer,egress,+ forwarder: Box::new(forwarder),defaults: defaults.into(),}),workers: Arc::new(DashMap::new()),@@ -139,9 +174,34 @@ where/// caller's span is captured: the returned future only awaits a oneshot, so/// the span has to travel with the command to reach the work itself.pub fn execute(+ &self,+ session_id: AgentSessionId,+ command: HarnessCommand,+ ) -> impl Future<Output = Result<()>> + Send + 'static {+ self.enqueue(session_id, command, true)+ }++ /// [`execute`](Self::execute), without resolving which replica manages+ /// the session first.+ ///+ /// The entry point for commands received *as* forwards: the sender+ /// already resolved management to this replica, and re-resolving here is+ /// what could bounce a command between two replicas whose lease views+ /// momentarily differ. Forwarding is single-hop; this is the second hop's+ /// half of that contract.+ pub fn execute_here(+ &self,+ session_id: AgentSessionId,+ command: HarnessCommand,+ ) -> impl Future<Output = Result<()>> + Send + 'static {+ self.enqueue(session_id, command, false)+ }++ fn enqueue(&self,session_id: AgentSessionId,mut command: HarnessCommand,+ route: bool,) -> impl Future<Output = Result<()>> + Send + 'static {let caller = tracing::Span::current();let result = loop {@@ -151,6 +211,7 @@ wherecommand,completed,span: caller.clone(),+ route,};match commands.send(queued) {@@ -238,6 +299,46 @@ where}}+/// The receiving half of command forwarding: what the internal forward route+/// calls, implemented by the harness as [`AgentHarnessService::execute_here`].+pub trait ForwardedCommands: Send + Sync + 'static {+ /// Run a command a peer already routed to this replica.+ fn execute_forwarded(+ &self,+ session_id: AgentSessionId,+ command: HarnessCommand,+ ) -> impl Future<Output = Result<()>> + Send;+}++impl<Sessions, Containers, Announcer, Runtimes, PromptContext, PromptComposer, Egress>+ ForwardedCommands+ for AgentHarnessService<+ Sessions,+ Containers,+ Announcer,+ Runtimes,+ PromptContext,+ PromptComposer,+ Egress,+ >+where+ Sessions: AgentSessionService,+ Containers: ContainerManager,+ Announcer: SessionAnnouncer,+ Runtimes: RuntimeConnections,+ PromptContext: ChannelPromptContext,+ PromptComposer: AgentPromptComposer,+ Egress: SandboxEgressProvisioner,+{+ async fn execute_forwarded(+ &self,+ session_id: AgentSessionId,+ command: HarnessCommand,+ ) -> Result<()> {+ self.execute_here(session_id, command).await+ }+}+/// The harness is what holds a session's live resources, so it is what the/// control routes notify. Both operations go through the per-session queue, so/// a teardown cannot land in the middle of an open and a model change cannot@@ -541,6 +642,55 @@ wherePromptComposer: AgentPromptComposer,Egress: SandboxEgressProvisioner,{+ /// Execute where the session's live actor is: locally when nobody (or+ /// this replica) manages it, on the managing peer otherwise.+ ///+ /// A failed forward re-reads the lease once: the one legitimate reason a+ /// live manager refuses its own session is that it died mid-flight, and+ /// then its heartbeat going stale is what lets this replica take over. A+ /// manager that is alive but unreachable stays an error - executing+ /// locally anyway is how two actors end up on one session.+ #[tracing::instrument(err, skip(self, command), fields(%session_id))]+ async fn route_then_execute(+ &self,+ session_id: AgentSessionId,+ command: HarnessCommand,+ ) -> Result<()> {+ // Open never routes: it is what creates the session row this routing+ // would read, and a fresh id has no manager to defer to.+ if matches!(command, HarnessCommand::Open(_)) {+ return self.execute(session_id, command).await;+ }+ let manager = match self.sessions.management(session_id).await? {+ SessionManagement::Unmanaged | SessionManagement::Ours => {+ return self.execute(session_id, command).await;+ }+ SessionManagement::Peer(manager) => manager,+ };+ let Some(address) = manager.address else {+ return Err(HarnessError::ManagerUnreachable(session_id));+ };+ tracing::info!(%session_id, peer = %manager.replica, "forwarding an agent session command");+ match self+ .forwarder+ .forward(&address, session_id, command.clone())+ .await+ {+ Ok(()) => Ok(()),+ Err(forward_error) => match self.sessions.management(session_id).await? {+ SessionManagement::Unmanaged | SessionManagement::Ours => {+ tracing::warn!(+ error = ?forward_error,+ %session_id,+ "the managing replica went stale mid-forward; executing locally"+ );+ self.execute(session_id, command).await+ }+ SessionManagement::Peer(_) => Err(forward_error),+ },+ }+ }+async fn execute(&self, session_id: AgentSessionId, command: HarnessCommand) -> Result<()> {match &command {HarnessCommand::Open(open)@@ -960,8 +1110,16 @@ async fn run_session_worker<command,completed,span,+ route,} = queued;- let result = inner.execute(session_id, command).instrument(span).await;+ let result = if route {+ inner+ .route_then_execute(session_id, command)+ .instrument(span)+ .await+ } else {+ inner.execute(session_id, command).instrument(span).await+ };let _ = completed.send(result);}}diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rsindex ec28d87289..291ba68017 100644--- a/crates/agent_harness/src/domain/service/test.rs+++ b/crates/agent_harness/src/domain/service/test.rs@@ -35,7 +35,9 @@ use crate::domain::model::{AgentKind, AnnounceOrigin, DeliverAction, HarnessCommand, HarnessDefaults, MentionOrigin,OpenSession, PriorChannelMessage, SessionDefaults, SpawnContainer,};-use crate::domain::ports::{AgentPromptComposer, ChannelPromptContext, ContainerManager as _};+use crate::domain::ports::{+ AgentPromptComposer, ChannelPromptContext, ContainerManager as _, NoPeers,+};use crate::outbound::runtime_registry::RuntimeRegistry;use crate::testing::helpers::agent::FakeAgent;use crate::testing::helpers::announcer::AnnouncerMock;@@ -210,6 +212,7 @@ fn harness_with_edges(prompt_context,prompt_composer,EgressProvisionerMock::new(),+ NoPeers,SessionDefaults {bot_id: BotId::TEST_A,model: "claude".to_owned(),@@ -1416,6 +1419,7 @@ async fn a_managed_session_opens_as_the_managed_default_bot() {PromptContextMock::default(),PromptComposerMock::default(),EgressProvisionerMock::new(),+ NoPeers,HarnessDefaults::new(SessionDefaults {bot_id: BotId::TEST_A,model: "claude".to_owned(),@@ -1801,3 +1805,157 @@ async fn set_sandbox_size_unsupported_does_not_persist() {);assert_eq!(containers.resumed(), 0);}++/// A [`CommandForwarder`] that records its calls and reports success.+#[derive(Clone, Default)]+struct RecordingForwarder {+ calls: Arc<Mutex<Vec<(String, AgentSessionId)>>>,+}++impl crate::domain::ports::CommandForwarder for RecordingForwarder {+ async fn forward(+ &self,+ target: &agent_session::domain::model::ReplicaAddress,+ session: AgentSessionId,+ _command: HarnessCommand,+ ) -> crate::domain::error::Result<()> {+ self.calls+ .lock()+ .unwrap()+ .push((target.as_str().to_owned(), session));+ Ok(())+ }+}++/// A forwarder standing in for a peer that died mid-forward: the call fails,+/// and by the time the caller re-reads the lease the peer's claim is gone.+#[derive(Clone)]+struct DyingPeerForwarder {+ repo: InMemoryAgentSessionRepo,+ claim: agent_session::domain::model::SessionClaim,+}++impl crate::domain::ports::CommandForwarder for DyingPeerForwarder {+ async fn forward(+ &self,+ _target: &agent_session::domain::model::ReplicaAddress,+ _session: AgentSessionId,+ _command: HarnessCommand,+ ) -> crate::domain::error::Result<()> {+ use agent_session::domain::ports::SessionOwnership as _;+ self.repo.release(&self.claim).await.expect("peer releases");+ Err(HarnessError::Forward(rootcause::report!(+ "the peer went away"+ )))+ }+}++/// Claim a session for a fabricated peer replica that publishes an address.+async fn claim_as_peer(+ repo: &InMemoryAgentSessionRepo,+ session: AgentSessionId,+ address: &str,+) -> agent_session::domain::model::SessionClaim {+ use agent_session::domain::model::{ClaimOutcome, ReplicaAddress, ReplicaId};+ use agent_session::domain::ports::SessionOwnership as _;+ let peer = ReplicaId::mint();+ repo.heartbeat(peer, Some(&ReplicaAddress::new(address)))+ .await+ .expect("peer heartbeats");+ match repo.claim(session, peer).await.expect("peer claims") {+ ClaimOutcome::Claimed(claim) => claim,+ ClaimOutcome::ManagedElsewhere(_) => panic!("nobody else should hold the test session"),+ }+}++/// A command for a session a live peer manages goes to the peer's address+/// and never executes here: the session outlives a Delete this replica would+/// otherwise have applied.+#[tokio::test]+async fn commands_for_a_peer_managed_session_forward_to_its_address() {+ let repo = InMemoryAgentSessionRepo::new();+ let session = agent_session::testing::test_agent_session(AgentSessionId::new());+ let id = session.id;+ repo.insert_session(session);+ let _claim = claim_as_peer(&repo, id, "http://10.0.0.7:8100").await;+ let forwarder = RecordingForwarder::default();+ let service = AgentHarnessService::new(+ AgentSessionServiceImpl::new(+ repo.clone(),+ FoldedMessageService::new(repo.clone()),+ NoOpRealtime,+ ),+ MockContainerManager::new(),+ AnnouncerMock::new(),+ RuntimeRegistry::<ContainerSender>::new(),+ PromptContextMock::default(),+ PromptComposerMock::default(),+ EgressProvisionerMock::new(),+ forwarder.clone(),+ SessionDefaults {+ bot_id: BotId::TEST_A,+ model: "claude".to_owned(),+ harness: "opencode".to_owned(),+ repo_url: "https://github.com/macro-inc/macro".to_owned(),+ },+ );++ service+ .execute(id, HarnessCommand::Delete)+ .await+ .expect("the forwarded command succeeds");++ assert_eq!(+ forwarder.calls.lock().unwrap().clone(),+ vec![("http://10.0.0.7:8100".to_owned(), id)]+ );+ assert!(+ repo.get(id).await.is_ok(),+ "the delete ran on the peer, not here"+ );+}++/// A forward that fails against a peer whose lease is gone by the re-read+/// falls back to executing locally, so a peer crash costs one retry rather+/// than the command.+#[tokio::test]+async fn a_dead_peers_command_falls_back_to_local_execution() {+ let repo = InMemoryAgentSessionRepo::new();+ let session = agent_session::testing::test_agent_session(AgentSessionId::new());+ let id = session.id;+ repo.insert_session(session);+ let claim = claim_as_peer(&repo, id, "http://10.0.0.7:8100").await;+ let service = AgentHarnessService::new(+ AgentSessionServiceImpl::new(+ repo.clone(),+ FoldedMessageService::new(repo.clone()),+ NoOpRealtime,+ ),+ MockContainerManager::new(),+ AnnouncerMock::new(),+ RuntimeRegistry::<ContainerSender>::new(),+ PromptContextMock::default(),+ PromptComposerMock::default(),+ EgressProvisionerMock::new(),+ DyingPeerForwarder {+ repo: repo.clone(),+ claim,+ },+ SessionDefaults {+ bot_id: BotId::TEST_A,+ model: "claude".to_owned(),+ harness: "opencode".to_owned(),+ repo_url: "https://github.com/macro-inc/macro".to_owned(),+ },+ );++ service+ .execute(id, HarnessCommand::Delete)+ .await+ .expect("the fallback executes locally");++ assert!(+ repo.get(id).await.is_err(),+ "the delete ran here once the peer was gone"+ );+}diff --git a/crates/agent_harness/src/inbound/forward.rs b/crates/agent_harness/src/inbound/forward.rsnew file mode 100644index 0000000000..6e428695f0--- /dev/null+++ b/crates/agent_harness/src/inbound/forward.rs@@ -0,0 +1,105 @@+//! The receiving half of replica-to-replica command forwarding.+//!+//! A peer that consumed a command for a session this replica manages POSTs it+//! here. Internal-only by construction: the caller is another replica of this+//! same deployment, authenticated with the shared internal key, never a user.+//! The handler executes without re-resolving management - forwarding is+//! single-hop, and this endpoint is the second hop honoring that contract.++use std::sync::Arc;++use agent_session::domain::model::AgentSessionId;+use axum::Router;+use axum::extract::{FromRef, Json, Path, State};+use axum::http::StatusCode;+use axum::response::{IntoResponse, Response};+use axum::routing::post;+use macro_authorization::{+ InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService, MacroAuthorizationState,+};+use macro_uuid::Uuid;++use crate::domain::error::HarnessError;+use crate::domain::model::HarnessCommand;+use crate::domain::service::ForwardedCommands;++/// State for the command-forwarding route.+pub struct ForwardGatewayState<Harness, Auth> {+ harness: Arc<Harness>,+ authorization_state: MacroAuthorizationState<Auth>,+}++impl<Harness, Auth> ForwardGatewayState<Harness, Auth> {+ /// Create forwarding state.+ pub fn new(harness: Arc<Harness>, authorization_state: MacroAuthorizationState<Auth>) -> Self {+ Self {+ harness,+ authorization_state,+ }+ }+}++// Manual Clone impl: everything is behind an Arc.+impl<Harness, Auth> Clone for ForwardGatewayState<Harness, Auth> {+ fn clone(&self) -> Self {+ Self {+ harness: Arc::clone(&self.harness),+ authorization_state: self.authorization_state.clone(),+ }+ }+}++impl<Harness, Auth> FromRef<ForwardGatewayState<Harness, Auth>> for MacroAuthorizationState<Auth> {+ fn from_ref(state: &ForwardGatewayState<Harness, Auth>) -> Self {+ state.authorization_state.clone()+ }+}++/// Build the router serving `POST /agent-sessions/{id}/command`. Mount it+/// under `/internal`.+pub fn forward_router<Harness, Auth, S>(state: ForwardGatewayState<Harness, Auth>) -> Router<S>+where+ Harness: ForwardedCommands,+ Auth: MacroAuthorizationService,+ S: Clone + Send + Sync + 'static,+{+ Router::new()+ .route(+ "/agent-sessions/{session_id}/command",+ post(forward_handler::<Harness, Auth>),+ )+ .with_state(state)+}++/// Execute a command a peer replica forwarded to this one.+async fn forward_handler<Harness, Auth>(+ State(state): State<ForwardGatewayState<Harness, Auth>>,+ _caller: MacroAuthorizationExtractor<Auth, InternalOnly>,+ Path(session_id): Path<Uuid>,+ Json(command): Json<HarnessCommand>,+) -> Response+where+ Harness: ForwardedCommands,+ Auth: MacroAuthorizationService,+{+ let session_id = AgentSessionId::new_from_uuid(session_id);+ match state.harness.execute_forwarded(session_id, command).await {+ Ok(()) => StatusCode::NO_CONTENT.into_response(),+ // Surfaced with its own status so the sender can tell "the actor is+ // not here after all" from an execution failure: the sender's+ // fallback re-reads the lease on any error, but a 409 is the signal+ // that its view was stale rather than that the command is bad.+ Err(HarnessError::Disconnected(id))+ | Err(HarnessError::Session(+ agent_session::domain::error::AgentSessionError::Disconnected(id),+ )) => (+ StatusCode::CONFLICT,+ format!("session {id} is not attached here"),+ )+ .into_response(),+ Err(error) => {+ tracing::error!(error = ?error, %session_id, "a forwarded command failed");+ (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response()+ }+ }+}diff --git a/crates/agent_harness/src/inbound/mod.rs b/crates/agent_harness/src/inbound/mod.rsindex 856caaa13c..aeac45aac8 100644--- a/crates/agent_harness/src/inbound/mod.rs+++ b/crates/agent_harness/src/inbound/mod.rs@@ -3,5 +3,6 @@//! Thin by construction - decode the broker message, ask the domain whether it//! is addressed to us, hand one value inward. No policy lives here.+pub mod forward;pub mod kafka;pub mod runtime_gateway;diff --git a/crates/agent_harness/src/outbound/forward.rs b/crates/agent_harness/src/outbound/forward.rsnew file mode 100644index 0000000000..87fb4c6d55--- /dev/null+++ b/crates/agent_harness/src/outbound/forward.rs@@ -0,0 +1,83 @@+//! HTTP implementation of [`CommandForwarder`]: one authenticated POST to the+//! managing replica's internal forward route, awaited for its result.+//!+//! Direct replica-to-replica, not through the load balancer: the target+//! address is the peer's own private URL as published with its heartbeat, so+//! the command lands on exactly the process holding the session's actor.++use agent_session::domain::model::{AgentSessionId, ReplicaAddress};+use reqwest::StatusCode;++use crate::domain::error::{HarnessError, Result};+use crate::domain::model::HarnessCommand;+use crate::domain::ports::CommandForwarder;++/// Header carrying the deployment's shared internal key, as+/// `macro_authorization`'s internal extractor reads it.+const INTERNAL_API_KEY_HEADER: &str = macro_authorization::INTERNAL_API_KEY_HEADER;++/// Forwards commands to peer replicas over their internal HTTP surface.+#[derive(Clone)]+pub struct HttpCommandForwarder {+ client: reqwest::Client,+ internal_api_key: String,+}++impl HttpCommandForwarder {+ /// A forwarder authenticating with the deployment's internal key.+ ///+ /// The timeout bounds the whole forwarded execution, not just the dial:+ /// the peer replies only once the command ran, and a Deliver can sit+ /// behind a sandbox resume, so this is generous rather than snappy.+ pub fn new(internal_api_key: String) -> Result<Self> {+ let client = reqwest::Client::builder()+ .connect_timeout(std::time::Duration::from_secs(2))+ .timeout(std::time::Duration::from_secs(120))+ .build()+ .map_err(|error| HarnessError::Forward(rootcause::report!(error).into_dynamic()))?;+ Ok(Self {+ client,+ internal_api_key,+ })+ }+}++impl CommandForwarder for HttpCommandForwarder {+ #[tracing::instrument(err, skip(self, command), fields(%target, %session))]+ async fn forward(+ &self,+ target: &ReplicaAddress,+ session: AgentSessionId,+ command: HarnessCommand,+ ) -> Result<()> {+ let url = format!(+ "{}/internal/agent-sessions/{}/command",+ target.as_str().trim_end_matches('/'),+ session+ );+ let response = self+ .client+ .post(url)+ .header(INTERNAL_API_KEY_HEADER, &self.internal_api_key)+ .json(&command)+ .send()+ .await+ .map_err(|error| HarnessError::Forward(rootcause::report!(error).into_dynamic()))?;+ let status = response.status();+ if status.is_success() {+ return Ok(());+ }+ let body = response.text().await.unwrap_or_default();+ // 409 is the peer saying "not attached here after all" - the caller's+ // stale-view fallback keys off any error, but keep the distinction+ // visible in the report for whoever reads the logs.+ let refused = if status == StatusCode::CONFLICT {+ "the peer refused the session"+ } else {+ "the peer failed the command"+ };+ Err(HarnessError::Forward(rootcause::report!(+ "{refused}: {status}: {body}"+ )))+ }+}diff --git a/crates/agent_harness/src/outbound/mod.rs b/crates/agent_harness/src/outbound/mod.rsindex 7fb25fdddb..6f7678e264 100644--- a/crates/agent_harness/src/outbound/mod.rs+++ b/crates/agent_harness/src/outbound/mod.rs@@ -8,6 +8,7 @@ pub mod containers;pub mod cursor;pub mod daytona;pub mod egress;+pub mod forward;pub mod local;pub(crate) mod managed_containers;pub mod namespace;diff --git a/crates/agent_session/src/domain/error.rs b/crates/agent_session/src/domain/error.rsindex 83f4afd230..3f79166aa6 100644--- a/crates/agent_session/src/domain/error.rs+++ b/crates/agent_session/src/domain/error.rs@@ -8,6 +8,10 @@ pub type Result<T, E = AgentSessionError> = std::result::Result<T, E>;pub enum AgentSessionError {#[error("agent session {0} already has an active transport")]AlreadyConnected(AgentSessionId),+ #[error("agent session {0} is managed by another live replica")]+ ManagedElsewhere(AgentSessionId),+ #[error("agent session {0} write was fenced out: another replica claimed the session")]+ FencedOut(AgentSessionId),#[error("acp handshake failed: {0}")]Handshake(String),#[error("agent session {0} is no longer connected")]diff --git a/crates/agent_session/src/domain/model.rs b/crates/agent_session/src/domain/model.rsindex 28cd843c8e..a3021121e4 100644--- a/crates/agent_session/src/domain/model.rs+++ b/crates/agent_session/src/domain/model.rs@@ -15,6 +15,123 @@ pub use agent_fold::domain::model::{Author, AuthorKind, FoldEvent, MessageId, OwnedFoldEvent, TurnId,};+/// Identity of one harness participant, minted fresh at construction.+///+/// A restarted process is a new replica: whatever the old identity claimed is+/// released by its heartbeat going stale, never inherited.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct ReplicaId(Uuid);++impl ReplicaId {+ /// Mint a fresh replica identity.+ #[must_use]+ pub fn mint() -> Self {+ Self(Uuid::new_v4())+ }++ /// The raw uuid, for persistence.+ #[must_use]+ pub fn as_uuid(&self) -> Uuid {+ self.0+ }++ /// Rebuild an identity from its persisted uuid.+ #[must_use]+ pub fn from_uuid(id: Uuid) -> Self {+ Self(id)+ }+}++impl std::fmt::Display for ReplicaId {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ self.0.fmt(f)+ }+}++/// A replica's own base URL, as peers should dial it for command forwarding.+///+/// Private-network address discovered by the replica itself at boot (the ECS+/// task metadata endpoint in deployments), published with its heartbeat.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct ReplicaAddress(String);++impl ReplicaAddress {+ /// Wrap a base URL, e.g. `http://10.0.1.7:8100`.+ #[must_use]+ pub fn new(address: impl Into<String>) -> Self {+ Self(address.into())+ }++ /// The base URL as a string.+ #[must_use]+ pub fn as_str(&self) -> &str {+ &self.0+ }+}++impl std::fmt::Display for ReplicaAddress {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ self.0.fmt(f)+ }+}++/// The live manager of a session, as read from the lease.+#[derive(Debug, Clone)]+pub struct SessionManager {+ /// The replica holding the claim.+ pub replica: ReplicaId,+ /// Where to forward its commands, when the replica has published one.+ /// `None` means the manager is live but unreachable - hold the error+ /// rather than execute somewhere the actor is not.+ pub address: Option<ReplicaAddress>,+}++/// Where a session's live actor runs, from one service instance's viewpoint.+#[derive(Debug, Clone)]+pub enum SessionManagement {+ /// No live replica manages the session; this instance may claim it by+ /// attaching, so commands execute locally.+ Unmanaged,+ /// This instance's replica manages it; commands execute locally.+ Ours,+ /// A live peer manages it; commands belong at its address.+ Peer(SessionManager),+}++/// A session's takeover counter, bumped by every successful claim.+///+/// Carried by the claim holder into each live-actor write; the store rejects+/// writes whose fence has been superseded, so a stale holder is neutralized+/// by the same statement that would have written (a fencing token).+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]+pub struct ManagerFence(pub i64);++/// Proof that this replica claimed a session's live management.+///+/// Obtained only from [`SessionOwnership::claim`](super::ports::SessionOwnership::claim);+/// holding one is what entitles an actor to attach and write the session's+/// log under its fence.+#[derive(Debug, Clone, Copy)]+pub struct SessionClaim {+ /// The claimed session.+ pub session: AgentSessionId,+ /// The replica holding the claim.+ pub replica: ReplicaId,+ /// The fence this claim writes under.+ pub fence: ManagerFence,+}++/// What claiming a session yielded.+#[derive(Debug, Clone, Copy)]+pub enum ClaimOutcome {+ /// This replica now manages the session.+ Claimed(SessionClaim),+ /// A replica with a fresh heartbeat already manages it. Until command+ /// forwarding exists this surfaces as an error; with it, commands are+ /// routed to the named replica instead.+ ManagedElsewhere(ReplicaId),+}+/// Display name assigned to a newly created agent session.pub const DEFAULT_AGENT_SESSION_NAME: &str = "Agent Session";diff --git a/crates/agent_session/src/domain/ports.rs b/crates/agent_session/src/domain/ports.rsindex 73eb13028f..7a1f321422 100644--- a/crates/agent_session/src/domain/ports.rs+++ b/crates/agent_session/src/domain/ports.rs@@ -272,6 +272,62 @@ pub trait ExternalSessionRepo: Send + Sync + 'static {fn delete(&self, id: AgentSessionId) -> impl Future<Output = Result<()>> + Send;}+/// How often a replica refreshes its heartbeat row.+pub const REPLICA_HEARTBEAT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(10);++/// How stale a replica's heartbeat may be before its claims are up for+/// grabs. Three missed heartbeats: long enough that one slow write does not+/// get a live replica's sessions stolen, short enough that a crashed+/// replica's sessions resume on the next prompt rather than minutes later.+pub const REPLICA_STALE_AFTER: std::time::Duration = std::time::Duration::from_secs(30);++/// The session-management lease: which replica holds a session's live actor.+///+/// Implemented by the same store that persists the session log, never+/// separately - the fence a claim carries is checked by that store's fenced+/// appends, and minting fences in one place while checking them in another+/// is the mis-wiring this coupling forbids.+///+/// Claiming is a single conditional update (compare-and-swap): it succeeds+/// when the session is unmanaged, already ours, or held by a replica whose+/// heartbeat has gone stale, and every success increments the session's+/// fence. There are deliberately no explicit locks anywhere in the contract -+/// see [`ManagerFence`](super::model::ManagerFence) for why a fence, not a+/// lock, is what neutralizes a stale holder.+pub trait SessionOwnership: Send + Sync + 'static {+ /// Claim live management of a session for `replica`, registering the+ /// replica's heartbeat as a side effect so a claim can never dangle on a+ /// replica the store has not seen.+ fn claim(+ &self,+ session: AgentSessionId,+ replica: ReplicaId,+ ) -> impl Future<Output = Result<ClaimOutcome>> + Send;++ /// Release a claim this replica holds. Conditional on the claim's fence+ /// still being current: releasing after having been superseded is a+ /// no-op, never a theft of the successor's claim. A session already+ /// released (or deleted) is in the asked-for state, so this succeeds.+ fn release(&self, claim: &SessionClaim) -> impl Future<Output = Result<()>> + Send;++ /// Refresh this replica's heartbeat, upserting its row and publishing+ /// `address` - the base URL peers forward this replica's sessions'+ /// commands to - when one is known.+ fn heartbeat(+ &self,+ replica: ReplicaId,+ address: Option<&ReplicaAddress>,+ ) -> impl Future<Output = Result<()>> + Send;++ /// The live manager of a session, if a replica with a fresh heartbeat+ /// holds its lease. `None` covers both an unclaimed session and one whose+ /// holder has gone stale - either way the session is claimable.+ fn manager_of(+ &self,+ session: AgentSessionId,+ ) -> impl Future<Output = Result<Option<SessionManager>>> + Send;+}+#[cfg_attr(feature = "test-utils", mockall::automock)]pub trait AgentSessionLogRepo: Send + Sync + 'static {/// Append a log entry and project any system event onto the session status.@@ -280,6 +336,21 @@ pub trait AgentSessionLogRepo: Send + Sync + 'static {log: AgentSessionLog,) -> impl Future<Output = Result<StoredAgentSessionLog>> + Send;+ /// [`create`](Self::create), conditioned on `claim` still holding the+ /// session's current fence.+ ///+ /// This is the write half of the fencing contract: the check and the+ /// append are one atomic statement, so a replica that stalled past its+ /// heartbeat and was superseded cannot interleave frames no matter when+ /// it wakes - its append matches nothing and fails with+ /// [`FencedOut`](super::error::AgentSessionError::FencedOut), which the+ /// actor treats as its cue to tear down.+ fn create_fenced(+ &self,+ log: AgentSessionLog,+ claim: &SessionClaim,+ ) -> impl Future<Output = Result<StoredAgentSessionLog>> + Send;+/// List all log entries for a session, in chronological order.////// Entries come back stamped with when the log recorded them: the framediff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rsindex 00e8132b6a..6cec76b604 100644--- a/crates/agent_session/src/domain/service.rs+++ b/crates/agent_session/src/domain/service.rs@@ -47,12 +47,13 @@ use super::connection::RuntimeAttachment;use super::error::{AgentSessionError, Result};use super::model::{AgentSession, AgentSessionId, AgentSessionLog, AgentSessionRenamed, AuthorKind, ChannelSession,- CreateAgentSessionParams, LogAppended, MAX_AGENT_SESSION_NAME_CHARS, Message, MessageId,- SandboxSize, SessionLog, StoredAgentSessionLog,+ ClaimOutcome, CreateAgentSessionParams, LogAppended, MAX_AGENT_SESSION_NAME_CHARS, Message,+ MessageId, ReplicaId, SandboxSize, SessionClaim, SessionLog, SessionManagement,+ StoredAgentSessionLog,};use super::ports::{AgentConnector, AgentSessionLogRepo, AgentSessionLogWriter, AgentSessionNameGenerator,- AgentSessionRealtime, AgentSessionRepo, NoOpAgentSessionNameGenerator,+ AgentSessionRealtime, AgentSessionRepo, NoOpAgentSessionNameGenerator, SessionOwnership,};use super::session::actors::{SessionActor, SessionCommand, Stepped};use super::session::{CloseReason, Input};@@ -148,6 +149,14 @@ pub trait AgentSessionService: Send + Sync + 'static {/// Persist that a session disconnected before a live actor could report it.fn mark_disconnected(&self, id: AgentSessionId) -> impl Future<Output = Result<()>> + Send;+ /// Where the session's live actor runs, from this instance's viewpoint:+ /// unmanaged (claimable here), ours, or a live peer's - in which case+ /// commands belong at the peer's address rather than in this process.+ fn management(+ &self,+ id: AgentSessionId,+ ) -> impl Future<Output = Result<SessionManagement>> + Send;+/// Attach a new transport to an existing persisted session.////// The attachment carries the connection's handshake gate as well as the@@ -230,6 +239,10 @@ pub struct AgentSessionServiceImpl<R, Folds, Rt, Namer = NoOpAgentSessionNameGenrealtime: Rt,name_generator: Namer,active: Arc<ActiveSessions>,+ /// This service's identity in the session-management lease. Minted at+ /// construction: a restarted process is a new replica, and its claims+ /// are recovered by heartbeat staleness, never inherited.+ replica: ReplicaId,tasks: TaskTracker,cancellation: CancellationToken,lifecycle: Arc<Mutex<()>>,@@ -249,6 +262,7 @@ impl<R, Folds, Rt> AgentSessionServiceImpl<R, Folds, Rt> {realtime,name_generator: NoOpAgentSessionNameGenerator,active: Arc::new(DashMap::new()),+ replica: ReplicaId::mint(),tasks: TaskTracker::new(),cancellation: CancellationToken::new(),lifecycle: Arc::new(Mutex::new(())),@@ -269,12 +283,33 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {realtime: self.realtime,name_generator,active: self.active,+ replica: self.replica,tasks: self.tasks,cancellation: self.cancellation,lifecycle: self.lifecycle,}}+ /// This service's identity in the session-management lease, for the+ /// composition root to heartbeat while the process lives.+ #[must_use]+ pub fn replica_id(&self) -> ReplicaId {+ self.replica+ }++ /// Adopt a caller-minted replica identity, replacing the constructor's.+ ///+ /// For processes with more than one attach-capable service instance: the+ /// lease's replica is the *process* (commands forward to an address, and+ /// every instance in a process shares one), so the composition root mints+ /// a single id and stamps it on each instance. The instances' session+ /// sets are disjoint by construction, so they never contend for a lease.+ #[must_use]+ pub fn with_replica(mut self, replica: ReplicaId) -> Self {+ self.replica = replica;+ self+ }+/// Stop active actors and wait for their tasks to release their transports.pub async fn shutdown(&self) {let lifecycle = self.lifecycle.lock().await;@@ -322,9 +357,10 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {session: AgentSession,attachment: RuntimeAttachment<Connector>,reservation: AttachReservation,+ claim: SessionClaim,) -> Result<()>where- R: AgentSessionRepo + AgentSessionLogRepo + Clone,+ R: AgentSessionRepo + AgentSessionLogRepo + SessionOwnership + Clone,Rt: AgentSessionRealtime + Clone + Send + Sync + 'static,Connector: AgentConnector,{@@ -348,8 +384,10 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {// rather than the bare repository - see module docs. Its fold starts// empty and catches itself up on the stored log on the first frame,// which costs an attach nothing until the session actually says- // something.- let logs = LiveSessionLogWriter::new(self.repo.clone(), self.realtime.clone());+ // something. Fenced under the claim taken above: if another replica+ // supersedes this one, the store rejects the next append and the+ // actor tears down through its ordinary log-failure path.+ let logs = LiveSessionLogWriter::fenced(self.repo.clone(), self.realtime.clone(), claim);let actor = SessionActor::new(id,session.acp_session_id,@@ -367,6 +405,8 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {marker,stopped_tx,self.cancellation.clone(),+ self.repo.clone(),+ claim,).with_current_subscriber(),);@@ -476,7 +516,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {impl<R, Folds, Rt, Namer> AgentSessionService for AgentSessionServiceImpl<R, Folds, Rt, Namer>where- R: AgentSessionRepo + AgentSessionLogRepo + Clone,+ R: AgentSessionRepo + AgentSessionLogRepo + SessionOwnership + Clone,Folds: FoldedMessageRepo + Clone + Send + Sync + 'static,Rt: AgentSessionRealtime + Clone + Send + Sync + 'static,Namer: AgentSessionNameGenerator + Clone,@@ -547,6 +587,14 @@ whereOk(())}+ async fn management(&self, id: AgentSessionId) -> Result<SessionManagement> {+ Ok(match self.repo.manager_of(id).await? {+ None => SessionManagement::Unmanaged,+ Some(manager) if manager.replica == self.replica => SessionManagement::Ours,+ Some(manager) => SessionManagement::Peer(manager),+ })+ }+async fn mark_disconnected(&self, id: AgentSessionId) -> Result<()> {let mut logs = LiveSessionLogWriter::new(self.repo.clone(), self.realtime.clone());tokio::time::timeout(@@ -571,10 +619,42 @@ wherewhereConnector: AgentConnector,{+ // Reservation first: it is the in-process exclusion, so no sibling+ // attach of this instance can race us to the claim below - which+ // matters because every successful claim bumps the fence, and bumping+ // it under a live actor of our own would fence that actor out.let reservation = self.reserve_attach(id).await?;- let session = self.repo.get(id).await?;- self.activate_reserved(session, attachment, reservation)- .await+ let claim = match self.repo.claim(id, self.replica).await? {+ ClaimOutcome::Claimed(claim) => claim,+ ClaimOutcome::ManagedElsewhere(holder) => {+ tracing::info!(%id, %holder, "agent session is managed by another live replica");+ return Err(AgentSessionError::ManagedElsewhere(id));+ }+ };+ let activated = async {+ let session = self.repo.get(id).await?;+ self.activate_reserved(session, attachment, reservation, claim)+ .await+ }+ .await;+ if let Err(error) = activated {+ // The actor that would have released this claim never started;+ // free it here so another replica is not left waiting out our+ // heartbeat to resume the session.+ self.repo+ .release(&claim)+ .await+ .inspect_err(|release_error| {+ tracing::error!(+ error = ?release_error,+ %id,+ "failed to release an agent session claim after a failed attach"+ );+ })+ .ok();+ return Err(error);+ }+ Ok(())}async fn send_action(@@ -740,6 +820,12 @@ pub struct LiveSessionLogWriter<R, Rt> {repo: R,realtime: Rt,fold: Option<FoldMachineImpl>,+ /// The management claim this writer appends under, when it has one. A+ /// session actor always writes fenced; the unfenced constructor exists+ /// for writers outside any live-management contest - `seed_jsonl`+ /// replaying a recording, and `mark_disconnected` recording that a+ /// runtime dropped before anything attached.+ claim: Option<SessionClaim>,}impl<R, Rt> LiveSessionLogWriter<R, Rt> {@@ -754,6 +840,18 @@ impl<R, Rt> LiveSessionLogWriter<R, Rt> {repo,realtime,fold: None,+ claim: None,+ }+ }++ /// [`new`](Self::new), with every append conditioned on `claim` still+ /// holding the session's current fence. What a live actor writes with.+ pub fn fenced(repo: R, realtime: Rt, claim: SessionClaim) -> Self {+ Self {+ repo,+ realtime,+ fold: None,+ claim: Some(claim),}}}@@ -783,7 +881,10 @@ where// Durable first: projections are rebuildable, but a frame omitted from// session history is not.- let stored = AgentSessionLogRepo::create(&self.repo, log.clone()).await?;+ let stored = match &self.claim {+ Some(claim) => self.repo.create_fenced(log.clone(), claim).await?,+ None => AgentSessionLogRepo::create(&self.repo, log.clone()).await?,+ };if let Some(fold) = &mut self.fold {let _ = fold.push(log.clone());@@ -967,16 +1068,20 @@ where}}-/// Step the actor until its machine stops, then release the registry entry.-async fn run_session<Connector, Logs>(+/// Step the actor until its machine stops, then release the registry entry+/// and the session's management claim.+async fn run_session<Connector, Logs, Ownership>(mut actor: SessionActor<Connector, Logs>,active: std::sync::Weak<ActiveSessions>,marker: Arc<()>,stopped: watch::Sender<bool>,cancellation: CancellationToken,+ ownership: Ownership,+ claim: SessionClaim,) whereConnector: AgentConnector,Logs: AgentSessionLogWriter + AgentSessionRepo,+ Ownership: SessionOwnership,{loop {let input = tokio::select! {@@ -997,6 +1102,13 @@ async fn run_session<Connector, Logs>(// Tear down the old transport before allowing another actor to attach.drop(actor);+ // Give the claim back before announcing the stop: fence-conditioned, so+ // if a successor already took over this quietly does nothing. Releasing+ // here rather than waiting for heartbeat staleness is what lets another+ // replica resume this session immediately after a graceful stop.+ if let Err(error) = ownership.release(&claim).await {+ tracing::error!(error = ?error, %id, "failed to release an agent session claim");+ }let _ = stopped.send(true);if let Some(active) = active.upgrade() {active.remove_if(&id, |_, current| {diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rsindex bc45b57572..e2ee17977f 100644--- a/crates/agent_session/src/domain/service/test.rs+++ b/crates/agent_session/src/domain/service/test.rs@@ -1,6 +1,8 @@use super::*;use crate::PROTOCOL_VERSION;-use crate::domain::model::{DEFAULT_AGENT_SESSION_NAME, Message, SessionBot};+use crate::domain::model::{+ DEFAULT_AGENT_SESSION_NAME, Message, ReplicaAddress, SessionBot, SessionManager,+};use crate::domain::ports::NoOpRealtime;use crate::domain::session::HandshakeStatus;use crate::testing::{InMemoryAgentSessionRepo, RecordingRealtime, test_agent_session};@@ -162,6 +164,21 @@ impl AgentSessionLogWriter for BlockingPromptLogs {}}+/// Claim a session's management for a freshly minted replica, as+/// `attach_session` does before activating.+async fn claim_for_test(repo: &InMemoryAgentSessionRepo, session: AgentSessionId) -> SessionClaim {+ match repo+ .claim(session, ReplicaId::mint())+ .await+ .expect("claim for test")+ {+ ClaimOutcome::Claimed(claim) => claim,+ ClaimOutcome::ManagedElsewhere(holder) => {+ panic!("test session is unexpectedly managed by {holder}")+ }+ }+}+fn owner_access(session: AgentSessionId) -> EntityAccessReceipt<OwnerAccessLevel> {EntityAccessReceipt::dangerously_assert_internal_user(&session.as_uuid().to_string(),@@ -400,7 +417,35 @@ impl AgentSessionRepo for BlockingPromptLogs {}}+/// Pure delegation: the lease semantics under test live in the shared+/// in-memory store, and this wrapper only intercepts log writes.+impl SessionOwnership for BlockingPromptLogs {+ async fn claim(&self, session: AgentSessionId, replica: ReplicaId) -> Result<ClaimOutcome> {+ self.repo.claim(session, replica).await+ }++ async fn release(&self, claim: &SessionClaim) -> Result<()> {+ self.repo.release(claim).await+ }++ async fn heartbeat(&self, replica: ReplicaId, address: Option<&ReplicaAddress>) -> Result<()> {+ self.repo.heartbeat(replica, address).await+ }++ async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {+ self.repo.manager_of(session).await+ }+}+impl AgentSessionLogRepo for BlockingPromptLogs {+ async fn create_fenced(+ &self,+ log: AgentSessionLog,+ claim: &SessionClaim,+ ) -> Result<StoredAgentSessionLog> {+ self.repo.create_fenced(log, claim).await+ }+async fn create(&self, log: AgentSessionLog) -> Result<StoredAgentSessionLog> {if self.hang_disconnect&& matches!(@@ -512,6 +557,7 @@ async fn close_claiming_an_attach_reservation_prevents_actor_start() {.await.expect("attach reserves before reading");let session = fx.repo.get(fx.session).await.expect("session exists");+ let claim = claim_for_test(&fx.repo, fx.session).await;let (stopped, marker) = fx.service.begin_stop(fx.session, false);let result = fx@@ -520,6 +566,7 @@ async fn close_claiming_an_attach_reservation_prevents_actor_start() {session,RuntimeAttachment::solo(PendingTransport),reservation,+ claim,).await;AgentSessionServiceImpl::<@@ -552,6 +599,7 @@ async fn shutdown_prevents_a_reserved_attach_from_spawning() {.await.expect("attach reserves before reading");let session = fx.repo.get(fx.session).await.expect("session exists");+ let claim = claim_for_test(&fx.repo, fx.session).await;fx.service.shutdown().await;let result = fx@@ -560,6 +608,7 @@ async fn shutdown_prevents_a_reserved_attach_from_spawning() {session,RuntimeAttachment::solo(PendingTransport),reservation,+ claim,).await;@@ -583,6 +632,30 @@ async fn close_does_not_remove_a_concurrent_delete_guard() {});}+/// The cross-replica half of what `AlreadyConnected` guards in-process: a+/// second service instance over the same store - two replicas, in production+/// - cannot attach a session whose managing replica is live. Its claim comes+/// back `ManagedElsewhere` and the attach refuses before touching the actor.+#[tokio::test]+async fn a_second_replica_cannot_attach_a_session_with_a_live_manager() {+ let fx = fixture();+ fx.service+ .attach_session(fx.session, RuntimeAttachment::solo(PendingTransport))+ .await+ .expect("first replica attaches");++ let second_replica = AgentSessionServiceImpl::new(+ fx.repo.clone(),+ FoldedMessageService::new(fx.repo.clone()),+ NoOpRealtime,+ );+ let result = second_replica+ .attach_session(fx.session, RuntimeAttachment::solo(PendingTransport))+ .await;++ assert!(matches!(result, Err(AgentSessionError::ManagedElsewhere(id)) if id == fx.session));+}+/// A command sent while the handshake never completes cannot hang its caller/// forever - see [`HANDSHAKE_TIMEOUT`]. Without that bound, this test would/// simply never finish. The actor it was stuck in cannot linger afterwards@@ -653,12 +726,15 @@ async fn cancellation_does_not_drop_an_effect_batch_after_machine_mutation() {let cancellation = CancellationToken::new();let marker = Arc::new(());let (stopped_tx, _) = watch::channel(false);+ let claim = claim_for_test(&repo, session).await;let task = tokio::spawn(run_session(actor,Arc::downgrade(&active),marker,stopped_tx,cancellation.clone(),+ repo.clone(),+ claim,));open_test_session(&inbound_tx, &mut outbound_rx, session).await;@@ -726,6 +802,7 @@ async fn live_inbound_logs_do_not_reuse_the_expired_handshake_deadline() {let active = Arc::new(ActiveSessions::new());let cancellation = CancellationToken::new();let (stopped_tx, _) = watch::channel(false);+ let claim = claim_for_test(&repo, session).await;let task = tokio::spawn(run_session(actor,@@ -733,6 +810,8 @@ async fn live_inbound_logs_do_not_reuse_the_expired_handshake_deadline() {Arc::new(()),stopped_tx,cancellation.clone(),+ repo.clone(),+ claim,).with_current_subscriber(),);diff --git a/crates/agent_session/src/outbound/postgres.rs b/crates/agent_session/src/outbound/postgres.rsindex d131311651..ab02baf791 100644--- a/crates/agent_session/src/outbound/postgres.rs+++ b/crates/agent_session/src/outbound/postgres.rs@@ -7,10 +7,14 @@ mod test;use crate::domain::error::{AgentSessionError, Result};use crate::domain::model::{- AgentSession, AgentSessionId, AgentSessionLog, ChannelSession, CreateAgentSessionParams,- ExternalSession, Message, SandboxSize, SessionBot, SessionStatus, StoredAgentSessionLog,+ AgentSession, AgentSessionId, AgentSessionLog, ChannelSession, ClaimOutcome,+ CreateAgentSessionParams, ExternalSession, ManagerFence, Message, ReplicaAddress, ReplicaId,+ SandboxSize, SessionBot, SessionClaim, SessionManager, SessionStatus, StoredAgentSessionLog,+};+use crate::domain::ports::{+ AgentSessionLogRepo, AgentSessionRepo, ExternalSessionRepo, REPLICA_STALE_AFTER,+ SessionOwnership,};-use crate::domain::ports::{AgentSessionLogRepo, AgentSessionRepo, ExternalSessionRepo};use crate::outbound::connection_gateway_realtime::SessionAudience;use agent_client_protocol::schema::v1::SessionId;use agent_runtime_protocol::domain::schema::v0::{SystemEvent, ToRuntimeMessage, ToServerMessage};@@ -767,6 +771,81 @@ impl AgentSessionLogRepo for PgAgentSessionRepo {})}+ async fn create_fenced(+ &self,+ log: AgentSessionLog,+ claim: &SessionClaim,+ ) -> Result<StoredAgentSessionLog> {+ let event_status = match &log.content {+ Message::ToServer(ToServerMessage::Event { event }) => {+ Some(SessionStatus::Event(event.clone()))+ }+ _ => None,+ };+ let (direction, content) = message_columns(&log.content)?;+ let mut transaction = self+ .pool+ .begin()+ .await+ .context("begin fenced agent session log create")?;+ // The fencing contract: the guard and the append are one statement,+ // so there is no instant between "checked we still hold the lease"+ // and "wrote" for a takeover to slip into.+ let created_at = sqlx::query_scalar!(+ r#"+ INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content)+ SELECT $1, $2, $3, $4, $5+ WHERE EXISTS (+ SELECT 1 FROM agent_session+ WHERE id = $2 AND manager_replica_id = $6 AND manager_fence = $7+ )+ RETURNING created_at+ "#,+ macro_uuid::generate_uuid_v7(),+ log.agent_session_id.as_uuid(),+ log.user_id.as_ref().map(|user_id| user_id.as_ref()),+ direction,+ content,+ claim.replica.as_uuid(),+ claim.fence.0,+ )+ .fetch_optional(&mut *transaction)+ .await+ .context("failed to create fenced agent session log entry")?;+ let Some(created_at) = created_at else {+ return Err(AgentSessionError::FencedOut(log.agent_session_id));+ };++ if let Some(status) = event_status {+ let (status, status_event_name) = status_columns(&status);+ sqlx::query!(+ r#"+ UPDATE agent_session+ SET status = $2,+ status_event_name = $3,+ modified_at = now()+ WHERE id = $1+ "#,+ log.agent_session_id.as_uuid(),+ status,+ status_event_name,+ )+ .execute(&mut *transaction)+ .await+ .context("failed to update agent session status from fenced log entry")?;+ }++ transaction+ .commit()+ .await+ .context("commit fenced agent session log create")?;++ Ok(StoredAgentSessionLog {+ created_at,+ entry: log,+ })+ }+async fn list_by_session(&self,agent_session_id: AgentSessionId,@@ -797,6 +876,144 @@ impl AgentSessionLogRepo for PgAgentSessionRepo {}}+impl SessionOwnership for PgAgentSessionRepo {+ async fn claim(&self, session: AgentSessionId, replica: ReplicaId) -> Result<ClaimOutcome> {+ // One statement: the replica's heartbeat row is upserted in the CTE+ // (a claim can never reference a replica the store has not seen),+ // then the lease itself is a compare-and-swap - taken only from+ // nobody, ourselves, or a stale holder, and every take bumps the+ // fence so a superseded holder's fenced writes stop matching.+ let fence = sqlx::query_scalar!(+ r#"+ WITH replica AS (+ INSERT INTO harness_replica (id, last_heartbeat_at)+ VALUES ($2, now())+ ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()+ )+ UPDATE agent_session+ SET manager_replica_id = $2,+ manager_fence = manager_fence + 1,+ modified_at = now()+ WHERE id = $1+ AND (+ manager_replica_id IS NULL+ OR manager_replica_id = $2+ OR NOT EXISTS (+ SELECT 1 FROM harness_replica live+ WHERE live.id = agent_session.manager_replica_id+ AND live.last_heartbeat_at > now() - make_interval(secs => $3)+ )+ )+ RETURNING manager_fence+ "#,+ session.as_uuid(),+ replica.as_uuid(),+ REPLICA_STALE_AFTER.as_secs_f64(),+ )+ .fetch_optional(&self.pool)+ .await+ .context("failed to claim agent session management")?;++ if let Some(fence) = fence {+ return Ok(ClaimOutcome::Claimed(SessionClaim {+ session,+ replica,+ fence: ManagerFence(fence),+ }));+ }++ // The swap matched nothing: either a live replica holds the lease, or+ // the session row is gone. A deleted session reads as Unknown so the+ // caller does not forward commands toward a row that no longer exists.+ let holder = sqlx::query_scalar!(+ r#"SELECT manager_replica_id FROM agent_session WHERE id = $1"#,+ session.as_uuid(),+ )+ .fetch_optional(&self.pool)+ .await+ .context("failed to read the agent session manager")?;+ match holder {+ Some(Some(holder)) => Ok(ClaimOutcome::ManagedElsewhere(ReplicaId::from_uuid(holder))),+ Some(None) | None => Err(AgentSessionError::Unknown(anyhow::anyhow!(+ "agent session {session} claim matched nothing yet no live replica holds it"+ ))),+ }+ }++ async fn release(&self, claim: &SessionClaim) -> Result<()> {+ // Conditional on both holder and fence: a release arriving after a+ // successor claimed must not free the successor's lease, and a+ // deleted session matches nothing, which is the asked-for state.+ sqlx::query!(+ r#"+ UPDATE agent_session+ SET manager_replica_id = NULL,+ modified_at = now()+ WHERE id = $1 AND manager_replica_id = $2 AND manager_fence = $3+ "#,+ claim.session.as_uuid(),+ claim.replica.as_uuid(),+ claim.fence.0,+ )+ .execute(&self.pool)+ .await+ .context("failed to release agent session management")?;+ Ok(())+ }++ async fn heartbeat(&self, replica: ReplicaId, address: Option<&ReplicaAddress>) -> Result<()> {+ // COALESCE keeps a previously published address when a beat carries+ // none, so a claim-created row filled in by one heartbeat is not+ // blanked by the next.+ sqlx::query!(+ r#"+ INSERT INTO harness_replica (id, last_heartbeat_at, address)+ VALUES ($1, now(), $2)+ ON CONFLICT (id) DO UPDATE+ SET last_heartbeat_at = now(),+ address = COALESCE(EXCLUDED.address, harness_replica.address)+ "#,+ replica.as_uuid(),+ address.map(ReplicaAddress::as_str),+ )+ .execute(&self.pool)+ .await+ .context("failed to heartbeat harness replica")?;+ // Housekeeping on the writer that is already here: rows a week past+ // their last heartbeat are boots nothing can still reference usefully+ // (their claims were stealable within seconds); the FK sets any+ // stragglers' claims to NULL.+ sqlx::query!(+ r#"DELETE FROM harness_replica WHERE last_heartbeat_at < now() - interval '7 days'"#,+ )+ .execute(&self.pool)+ .await+ .context("failed to prune stale harness replicas")?;+ Ok(())+ }++ async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {+ let row = sqlx::query!(+ r#"+ SELECT live.id AS replica_id, live.address+ FROM agent_session+ JOIN harness_replica live ON live.id = agent_session.manager_replica_id+ WHERE agent_session.id = $1+ AND live.last_heartbeat_at > now() - make_interval(secs => $2)+ "#,+ session.as_uuid(),+ REPLICA_STALE_AFTER.as_secs_f64(),+ )+ .fetch_optional(&self.pool)+ .await+ .context("failed to read the agent session's live manager")?;+ Ok(row.map(|row| SessionManager {+ replica: ReplicaId::from_uuid(row.replica_id),+ address: row.address.map(ReplicaAddress::new),+ }))+ }+}+/// Folding reads the log through `agent_fold`'s own port; this adapter/// already speaks [`AgentSessionLogRepo`], so bridging is one line.impl agent_fold::domain::ports::LogRepo for PgAgentSessionRepo {diff --git a/crates/agent_session/src/outbound/postgres/test.rs b/crates/agent_session/src/outbound/postgres/test.rsindex 61e917cdc0..1d20c178c7 100644--- a/crates/agent_session/src/outbound/postgres/test.rs+++ b/crates/agent_session/src/outbound/postgres/test.rs@@ -892,3 +892,138 @@ async fn deleting_a_session_cascades_its_external_row(pool: PgPool) {None);}++/// Backdate a replica's heartbeat far past `REPLICA_STALE_AFTER`, so its+/// claims read as up for grabs.+async fn let_heartbeat_go_stale(pool: &PgPool, replica: ReplicaId) {+ sqlx::query!(+ r#"UPDATE harness_replica SET last_heartbeat_at = now() - interval '10 minutes' WHERE id = $1"#,+ replica.as_uuid(),+ )+ .execute(pool)+ .await+ .expect("backdate replica heartbeat");+}++fn claimed(outcome: ClaimOutcome) -> SessionClaim {+ match outcome {+ ClaimOutcome::Claimed(claim) => claim,+ ClaimOutcome::ManagedElsewhere(holder) => {+ panic!("expected to claim, but {holder} manages the session")+ }+ }+}++fn fenced_log(id: AgentSessionId) -> AgentSessionLog {+ AgentSessionLog {+ agent_session_id: id,+ user_id: None,+ content: Message::ToRuntime(ToRuntimeMessage::Acp(acp_notification())),+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn claiming_is_reentrant_and_every_claim_bumps_the_fence(pool: PgPool) {+ let repo = PgAgentSessionRepo::new(pool.clone());+ let bot_id = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot_id, None, None)).await;+ let replica = ReplicaId::mint();++ let first = claimed(repo.claim(session.id, replica).await.expect("first claim"));+ let second = claimed(repo.claim(session.id, replica).await.expect("second claim"));++ assert_eq!(first.fence, ManagerFence(1));+ assert_eq!(second.fence, ManagerFence(2));+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_live_holder_blocks_a_second_replica(pool: PgPool) {+ let repo = PgAgentSessionRepo::new(pool.clone());+ let bot_id = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot_id, None, None)).await;+ let holder = ReplicaId::mint();+ let contender = ReplicaId::mint();++ claimed(repo.claim(session.id, holder).await.expect("claim"));+ match repo.claim(session.id, contender).await.expect("contend") {+ ClaimOutcome::ManagedElsewhere(seen) => assert_eq!(seen, holder),+ ClaimOutcome::Claimed(_) => panic!("a live holder's claim was stolen"),+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_stale_holder_is_superseded(pool: PgPool) {+ let repo = PgAgentSessionRepo::new(pool.clone());+ let bot_id = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot_id, None, None)).await;+ let crashed = ReplicaId::mint();+ let successor = ReplicaId::mint();++ claimed(repo.claim(session.id, crashed).await.expect("claim"));+ let_heartbeat_go_stale(&pool, crashed).await;++ let takeover = claimed(repo.claim(session.id, successor).await.expect("takeover"));+ assert_eq!(takeover.fence, ManagerFence(2));+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn release_frees_the_lease_but_never_a_successors(pool: PgPool) {+ let repo = PgAgentSessionRepo::new(pool.clone());+ let bot_id = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot_id, None, None)).await;+ let crashed = ReplicaId::mint();+ let successor = ReplicaId::mint();+ let third = ReplicaId::mint();++ let superseded = claimed(repo.claim(session.id, crashed).await.expect("claim"));+ let_heartbeat_go_stale(&pool, crashed).await;+ let current = claimed(repo.claim(session.id, successor).await.expect("takeover"));++ // The superseded holder's release is a no-op: the successor still holds.+ repo.release(&superseded).await.expect("stale release");+ match repo.claim(session.id, third).await.expect("contend") {+ ClaimOutcome::ManagedElsewhere(seen) => assert_eq!(seen, successor),+ ClaimOutcome::Claimed(_) => panic!("a stale release freed the successor's lease"),+ }++ // The current holder's release frees it for anyone.+ repo.release(¤t).await.expect("current release");+ claimed(repo.claim(session.id, third).await.expect("reclaim"));+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_fenced_append_rejects_a_superseded_writer(pool: PgPool) {+ let repo = PgAgentSessionRepo::new(pool.clone());+ let bot_id = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot_id, None, None)).await;+ let zombie = ReplicaId::mint();+ let successor = ReplicaId::mint();++ let old_claim = claimed(repo.claim(session.id, zombie).await.expect("claim"));+ repo.create_fenced(fenced_log(session.id), &old_claim)+ .await+ .expect("the live holder's fenced append lands");++ let_heartbeat_go_stale(&pool, zombie).await;+ let new_claim = claimed(repo.claim(session.id, successor).await.expect("takeover"));++ // The zombie wakes and writes under its superseded fence: rejected by the+ // same statement that would have written, no matter how it got confused.+ match repo.create_fenced(fenced_log(session.id), &old_claim).await {+ Err(AgentSessionError::FencedOut(id)) => assert_eq!(id, session.id),+ other => panic!("a superseded fence wrote anyway: {other:?}"),+ }++ repo.create_fenced(fenced_log(session.id), &new_claim)+ .await+ .expect("the successor's fenced append lands");++ let log = AgentSessionLogRepo::list_by_session(&repo, session.id)+ .await+ .expect("list log");+ assert_eq!(+ log.len(),+ 2,+ "exactly the two live-holder appends are in the log"+ );+}diff --git a/crates/agent_session/src/testing.rs b/crates/agent_session/src/testing.rsindex 622f2aaf1b..578476df99 100644--- a/crates/agent_session/src/testing.rs+++ b/crates/agent_session/src/testing.rs@@ -6,11 +6,15 @@use crate::domain::error::{AgentSessionError, Result};use crate::domain::model::{- AgentSession, AgentSessionId, AgentSessionLog, ChannelSession, CreateAgentSessionParams,- DEFAULT_AGENT_SESSION_NAME, LogAppended, SandboxSize, SessionBot, SessionStatus,- StoredAgentSessionLog,+ AgentSession, AgentSessionId, AgentSessionLog, ChannelSession, ClaimOutcome,+ CreateAgentSessionParams, DEFAULT_AGENT_SESSION_NAME, LogAppended, ManagerFence,+ ReplicaAddress, ReplicaId, SandboxSize, SessionBot, SessionClaim, SessionManager,+ SessionStatus, StoredAgentSessionLog,+};+use crate::domain::ports::{+ AgentSessionLogRepo, AgentSessionRealtime, AgentSessionRepo, REPLICA_STALE_AFTER,+ SessionOwnership,};-use crate::domain::ports::{AgentSessionLogRepo, AgentSessionRealtime, AgentSessionRepo};use agent_client_protocol::schema::v1::SessionId;use agent_runtime_protocol::domain::schema::v0::ToServerMessage;use bots::domain::models::BotId;@@ -20,6 +24,13 @@ use std::collections::HashMap;use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::{Arc, Mutex};+/// One session's lease state: the holding replica (if any) and the fence,+/// which outlives the holder as in the real schema.+type Lease = (Option<ReplicaId>, i64);++/// One replica's row: its last heartbeat and published forwarding address.+type ReplicaRow = (std::time::Instant, Option<ReplicaAddress>);+/// An in-memory [`AgentSessionRepo`] and [`AgentSessionLogRepo`].////// Cheap to clone - clones share one store, so a handle kept for assertions@@ -36,6 +47,11 @@ pub struct InMemoryAgentSessionRepo {user_sizes: Arc<Mutex<HashMap<String, SandboxSize>>>,log_reads: Arc<AtomicUsize>,session_reads: Arc<AtomicUsize>,+ /// Replica heartbeats and published addresses, mirroring `harness_replica`.+ replicas: Arc<Mutex<HashMap<ReplicaId, ReplicaRow>>>,+ /// Session -> lease, mirroring the lease columns: release clears the+ /// holder and leaves the counter.+ leases: Arc<Mutex<HashMap<AgentSessionId, Lease>>>,}impl InMemoryAgentSessionRepo {@@ -318,6 +334,99 @@ impl AgentSessionRepo for InMemoryAgentSessionRepo {}}+impl SessionOwnership for InMemoryAgentSessionRepo {+ async fn claim(&self, session: AgentSessionId, replica: ReplicaId) -> Result<ClaimOutcome> {+ if !self+ .sessions+ .lock()+ .expect("in-memory session store is not poisoned")+ .contains_key(&session)+ {+ return Err(AgentSessionError::Unknown(anyhow::anyhow!(+ "agent session {session} does not exist to claim"+ )));+ }+ let now = std::time::Instant::now();+ let mut replicas = self+ .replicas+ .lock()+ .expect("in-memory replica store is not poisoned");+ replicas.entry(replica).or_insert((now, None)).0 = now;+ let mut leases = self+ .leases+ .lock()+ .expect("in-memory lease store is not poisoned");+ let (holder, fence) = leases.entry(session).or_insert((None, 0));+ let holder_is_live = holder.filter(|holder| *holder != replica).filter(|holder| {+ replicas+ .get(holder)+ .is_some_and(|(beat, _)| now.duration_since(*beat) < REPLICA_STALE_AFTER)+ });+ if let Some(holder) = holder_is_live {+ return Ok(ClaimOutcome::ManagedElsewhere(holder));+ }+ *holder = Some(replica);+ *fence += 1;+ Ok(ClaimOutcome::Claimed(SessionClaim {+ session,+ replica,+ fence: ManagerFence(*fence),+ }))+ }++ async fn release(&self, claim: &SessionClaim) -> Result<()> {+ let mut leases = self+ .leases+ .lock()+ .expect("in-memory lease store is not poisoned");+ if let Some((holder, fence)) = leases.get_mut(&claim.session)+ && *holder == Some(claim.replica)+ && *fence == claim.fence.0+ {+ *holder = None;+ }+ Ok(())+ }++ async fn heartbeat(&self, replica: ReplicaId, address: Option<&ReplicaAddress>) -> Result<()> {+ let mut replicas = self+ .replicas+ .lock()+ .expect("in-memory replica store is not poisoned");+ let entry = replicas+ .entry(replica)+ .or_insert((std::time::Instant::now(), None));+ entry.0 = std::time::Instant::now();+ // As in the real adapter: a beat carrying no address keeps the one+ // already published.+ if let Some(address) = address {+ entry.1 = Some(address.clone());+ }+ Ok(())+ }++ async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {+ let leases = self+ .leases+ .lock()+ .expect("in-memory lease store is not poisoned");+ let Some((Some(holder), _)) = leases.get(&session) else {+ return Ok(None);+ };+ let replicas = self+ .replicas+ .lock()+ .expect("in-memory replica store is not poisoned");+ Ok(replicas+ .get(holder)+ .filter(|(beat, _)| beat.elapsed() < REPLICA_STALE_AFTER)+ .map(|(_, address)| SessionManager {+ replica: *holder,+ address: address.clone(),+ }))+ }+}+impl AgentSessionLogRepo for InMemoryAgentSessionRepo {async fn create(&self, log: AgentSessionLog) -> Result<StoredAgentSessionLog> {let model_change = match &log.content {@@ -367,6 +476,28 @@ impl AgentSessionLogRepo for InMemoryAgentSessionRepo {Ok(stored)}+ async fn create_fenced(+ &self,+ log: AgentSessionLog,+ claim: &SessionClaim,+ ) -> Result<StoredAgentSessionLog> {+ let fenced_out = {+ let leases = self+ .leases+ .lock()+ .expect("in-memory lease store is not poisoned");+ !matches!(+ leases.get(&log.agent_session_id),+ Some((holder, fence))+ if *holder == Some(claim.replica) && *fence == claim.fence.0+ )+ };+ if fenced_out {+ return Err(AgentSessionError::FencedOut(log.agent_session_id));+ }+ AgentSessionLogRepo::create(self, log).await+ }+async fn list_by_session(&self,agent_session_id: AgentSessionId,diff --git a/crates/macro_db_client/migrations/20260828161717_agent_session_manager_lease.sql b/crates/macro_db_client/migrations/20260828161717_agent_session_manager_lease.sqlnew file mode 100644index 0000000000..f46553fe8b--- /dev/null+++ b/crates/macro_db_client/migrations/20260828161717_agent_session_manager_lease.sql@@ -0,0 +1,22 @@+-- Which harness replica holds a session's live actor, as a lease.+--+-- One row per booted harness participant, heartbeated while it lives. A+-- replica that stops heartbeating is dead by definition: its claims become+-- claimable by anyone, so a crashed process releases everything at once+-- without per-session cleanup.+CREATE TABLE harness_replica (+ id UUID PRIMARY KEY,+ started_at TIMESTAMPTZ NOT NULL DEFAULT now(),+ last_heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now()+);++-- The lease itself lives on the session so claiming and deleting are+-- transactional with the row they protect. manager_fence is a takeover+-- counter: every successful claim increments it, and live-actor log appends+-- are conditioned on the fence they were attached under, so a replica that+-- stalls past its heartbeat and gets superseded has its writes rejected by+-- the store rather than interleaved (fencing tokens). The fence never+-- resets - ON DELETE SET NULL frees the claim but keeps the counter.+ALTER TABLE agent_session+ ADD COLUMN manager_replica_id UUID REFERENCES harness_replica(id) ON DELETE SET NULL,+ ADD COLUMN manager_fence BIGINT NOT NULL DEFAULT 0;diff --git a/crates/macro_db_client/migrations/20260828183712_harness_replica_address.sql b/crates/macro_db_client/migrations/20260828183712_harness_replica_address.sqlnew file mode 100644index 0000000000..f63058ae4b--- /dev/null+++ b/crates/macro_db_client/migrations/20260828183712_harness_replica_address.sql@@ -0,0 +1,8 @@+-- Where to forward a session's commands when another replica manages it:+-- the replica's own private base URL, written with its heartbeat. Nullable+-- because a replica row can be created by a claim before the first+-- heartbeat carries the address, and because a local single-replica stack+-- has no routable address to publish - a peer that reads NULL treats the+-- manager as unreachable rather than guessing.+ALTER TABLE harness_replica+ ADD COLUMN address TEXT;diff --git a/infra/stacks/agent-harness-service/agent_harness_service.ts b/infra/stacks/agent-harness-service/agent_harness_service.tsindex 794583e615..8e89628c2e 100644--- a/infra/stacks/agent-harness-service/agent_harness_service.ts+++ b/infra/stacks/agent-harness-service/agent_harness_service.ts@@ -59,12 +59,25 @@ type Args = {};/**- * The agent harness service. Like agent-proxy before it, this component pins- * `desiredCount` to 1 and forces a stop-then-start deployment (min healthy- * 0%, max 100%) with no autoscaling: the harness owns the live agent-session- * actors in process memory with no cross-instance sync, and its Kafka consumer- * groups must not split partitions across two momentarily-coexisting tasks- * (see the consumer groups in `services/agent_harness_service`).+ * The agent harness service. Replicated in every environment: each replica+ * claims the sessions whose live actors it holds through a fenced Postgres+ * lease, and a command landing on the wrong replica is forwarded task-to-task+ * over the private network (`POST /internal/agent-sessions/{id}/command`,+ * internal-key authenticated) - which is why the service security group+ * allows itself ingress on the service port. The Kafka consumer group splits+ * partitions across live tasks; the lease plus forwarding is what makes that+ * split correct.+ *+ * Deploys roll (min healthy 100%, max 200%): the outgoing tasks keep serving+ * until their replacements pass health checks, so the control API and egress+ * proxy stay up instead of blacking out for the old tasks' sandbox cleanup.+ * A deploy therefore doubles the replica count for its duration rather than+ * introducing coexistence - replicas coexist all the time, and the lease is+ * what makes both the steady state and the overlap correct. A command routed+ * to a new task for a session whose live actor is still on an old one+ * self-heals through resume. Live sessions already restart across every+ * deploy (`shutdown_all` stops each sandbox), so the overlap narrows the+ * blast radius rather than widening it.*/export class AgentHarnessService extends pulumi.ComponentResource {public role: aws.iam.Role;@@ -310,18 +323,20 @@ export class AgentHarnessService extends pulumi.ComponentResource {enable: true,rollback: true,},- // Never run 2 tasks at once, even transiently during a deploy: stop- // the old one before the new one starts (see the class doc comment).- // This blackout covers the egress proxy too - sandbox git and MCP- // calls fail for the whole window, they are not more available than- // the control API.- deploymentMinimumHealthyPercent: 0,- deploymentMaximumPercent: 100,- desiredCount: 1,+ // Rolling replace: the old task serves until the new one is healthy+ // (see the class doc comment for the overlap semantics). A failed+ // deploy rolls back with the old task still up rather than at zero.+ deploymentMinimumHealthyPercent: 100,+ deploymentMaximumPercent: 200,+ // Every environment runs two, so dev stays prod-shaped: forwarding is+ // dead code at one task (management() can only answer Ours or+ // Unmanaged), and a path only dev never exercises is one whose+ // regressions surface in prod.+ desiredCount: 2,// ALB checks /health every 10s and fails the target after two// misses (~20s). HTTP does not listen until after DB, AWS config,// and JWT secrets, so a 0s grace period trips the circuit breaker- // on this stop-then-start replace. Ignore those checks until bind.+ // before the replacement binds. Ignore those checks until then.healthCheckGracePeriodSeconds: 120,taskDefinitionArgs: {taskRole: {@@ -473,6 +488,24 @@ export class AgentHarnessService extends pulumi.ComponentResource {{ parent: this });+ // Replica-to-replica command forwarding: a task that consumed a command+ // for a session another replica manages POSTs it directly to that+ // replica's private address, bypassing the load balancer.+ new aws.vpc.SecurityGroupIngressRule(+ `${BASE_NAME}-peer-in`,+ {+ securityGroupId: serviceSg.id,+ description:+ 'Allow harness replicas to forward session commands to each other',+ referencedSecurityGroupId: serviceSg.id,+ fromPort: serviceContainerPort,+ toPort: serviceContainerPort,+ ipProtocol: 'tcp',+ tags: this.tags,+ },+ { parent: this }+ );+new aws.vpc.SecurityGroupEgressRule(`${BASE_NAME}-service-out`,{diff --git a/services/agent_harness_service/Cargo.toml b/services/agent_harness_service/Cargo.tomlindex 0864294d16..6baa4e0601 100644--- a/services/agent_harness_service/Cargo.toml+++ b/services/agent_harness_service/Cargo.toml@@ -61,6 +61,7 @@ macro_authorization = { path = "../../crates/macro_authorization", default-featu] }macro_cors = { path = "../../crates/macro_cors" }secretsmanager_client = { path = "../../crates/secretsmanager_client" }+reqwest = { workspace = true }serde_json = { workspace = true }utoipa = { workspace = true }utoipa-swagger-ui = { workspace = true }diff --git a/services/agent_harness_service/src/api.rs b/services/agent_harness_service/src/api.rsindex b9739d5506..ddf3807c2f 100644--- a/services/agent_harness_service/src/api.rs+++ b/services/agent_harness_service/src/api.rs@@ -7,6 +7,8 @@ use std::time::Duration;use agent_egress::domain::service::EgressService;use agent_egress::inbound::axum_router::{EgressRouterState, egress_router};+use agent_harness::domain::service::ForwardedCommands;+use agent_harness::inbound::forward::{ForwardGatewayState, forward_router};use agent_harness::inbound::runtime_gateway::{RuntimeGatewayState, runtime_gateway_router};use agent_session::domain::ports::{AgentSessionNotificationRecipient, BotDirectory, SessionOpener,@@ -55,11 +57,12 @@ where}/// Build the router and serve it until the process is asked to stop.-pub async fn setup_and_serve<T, R, Opener, Bots, Access, Auth>(+pub async fn setup_and_serve<T, R, Opener, Bots, Access, Auth, Harness>(read_state: AgentSessionRouterState<T, Access, Auth>,control_state: AgentSessionControlState<R, Access, Auth>,create_state: CreateSessionState<Opener, Bots, Auth>,gateway_state: RuntimeGatewayState<Bots, Auth>,+ forward_state: ForwardGatewayState<Harness, Auth>,port: u16,shutdown: impl Future<Output = ()> + Send + 'static,) -> anyhow::Result<()>@@ -70,12 +73,19 @@ whereBots: BotDirectory,Access: EntityAccessService,Auth: MacroAuthorizationService,+ Harness: ForwardedCommands,{- let app = api_router(read_state, control_state, create_state, gateway_state)- .layer(MacroRequestIdAndTracingLayer::new(Duration::from_millis(200)).into_inner())- .merge(Router::new().route("/health", get(health)))- .layer(macro_cors::cors_layer())- .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()));+ let app = api_router(+ read_state,+ control_state,+ create_state,+ gateway_state,+ forward_state,+ )+ .layer(MacroRequestIdAndTracingLayer::new(Duration::from_millis(200)).into_inner())+ .merge(Router::new().route("/health", get(health)))+ .layer(macro_cors::cors_layer())+ .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()));let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}")).await@@ -89,11 +99,12 @@ where.context("agent harness service http failed")}-fn api_router<T, R, Opener, Bots, Access, Auth>(+fn api_router<T, R, Opener, Bots, Access, Auth, Harness>(read_state: AgentSessionRouterState<T, Access, Auth>,control_state: AgentSessionControlState<R, Access, Auth>,create_state: CreateSessionState<Opener, Bots, Auth>,gateway_state: RuntimeGatewayState<Bots, Auth>,+ forward_state: ForwardGatewayState<Harness, Auth>,) -> RouterwhereT: AgentSessionService,@@ -102,6 +113,7 @@ whereBots: BotDirectory,Access: EntityAccessService,Auth: MacroAuthorizationService,+ Harness: ForwardedCommands,{let agent_sessions = agent_session_read_router(read_state.clone()).merge(agent_session_control_router(control_state))@@ -110,6 +122,11 @@ where.nest("/agent-sessions", agent_sessions).merge(agent_sandbox_size_router(read_state)).nest("/runtime", runtime_gateway_router(gateway_state))+ // Replica-to-replica command forwarding. Internal-key authenticated,+ // and reached over task-to-task networking rather than the load+ // balancer; mounting it on the public listener is fine because the+ // extractor refuses anything without the deployment's internal key.+ .nest("/internal", forward_router(forward_state))}async fn health() -> &'static str {diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rsindex e64412d357..9da1972d60 100644--- a/services/agent_harness_service/src/main.rs+++ b/services/agent_harness_service/src/main.rs@@ -24,6 +24,7 @@ use agent_egress::outbound::session_authority::StoredTokenSessionAuthority;use agent_fold::domain::service::FoldedMessageService;use agent_harness::domain::model::{HarnessCommand, HarnessDefaults, SessionDefaults};use agent_harness::domain::service::AgentHarnessService;+use agent_harness::inbound::forward::ForwardGatewayState;use agent_harness::inbound::kafka::{RoutedTrigger, route_agent_trigger};use agent_harness::inbound::runtime_gateway::RuntimeGatewayState;use agent_harness::outbound::agent_prompt_composer::LexicalAgentPromptComposer;@@ -36,13 +37,15 @@ use agent_harness::outbound::daytona::{DaytonaContainerManager, DaytonaSettings, Snapshot,};use agent_harness::outbound::egress::EgressProvisioner;+use agent_harness::outbound::forward::HttpCommandForwarder;use agent_harness::outbound::local::{LocalContainerManager, LocalSettings};use agent_harness::outbound::routing::RoutedContainerManager;use agent_harness::outbound::runtime_registry::RuntimeRegistry;use agent_inmem::outbound::log_frames::LogFrameSource;use agent_inmem::outbound::manager::InMemAgentManager;use agent_inmem::outbound::rig_engine::RigTurnEngine;-use agent_session::domain::ports::NoOpRealtime;+use agent_session::domain::model::{ReplicaAddress, ReplicaId};+use agent_session::domain::ports::{NoOpRealtime, SessionOwnership as _};use agent_session::domain::service::AgentSessionServiceImpl;use agent_session::inbound::axum_router::{AgentSessionControlState, AgentSessionRouterState, CreateSessionState,@@ -125,6 +128,63 @@ struct PendingHarnessWork {description: &'static str,}+/// This task's own private base URL, for peers forwarding session commands.+///+/// On ECS the task metadata endpoint (v4) publishes the task's private IPv4;+/// the port is the service listener peers can reach through the task-to-task+/// security group rule. Local development has no routable address and runs+/// one replica, so `None` is correct there; every other environment refuses+/// to boot rather than heartbeat an unreachable identity, because a replica+/// without an address is one whose sessions peers can see but never reach.+async fn discover_replica_address(+ environment: Environment,+ port: u16,+) -> anyhow::Result<Option<agent_session::domain::model::ReplicaAddress>> {+ // Explicit override first: how a local multi-replica experiment names+ // each process (`REPLICA_ADDRESS=http://127.0.0.1:<port>`), since nothing+ // off ECS can discover an address.+ if let Some(address) = macro_env_var::optional_read_env_var("REPLICA_ADDRESS")+ .ok()+ .flatten()+ {+ return Ok(Some(agent_session::domain::model::ReplicaAddress::new(+ address,+ )));+ }+ let Some(metadata_uri) = macro_env_var::optional_read_env_var("ECS_CONTAINER_METADATA_URI_V4")+ .ok()+ .flatten()+ else {+ if matches!(environment, Environment::Local) {+ return Ok(None);+ }+ anyhow::bail!(+ "ECS_CONTAINER_METADATA_URI_V4 is unset outside a local environment: \+ this replica would be unreachable for command forwarding"+ );+ };+ let task: serde_json::Value = reqwest::Client::new()+ .get(format!("{metadata_uri}/task"))+ .timeout(std::time::Duration::from_secs(5))+ .send()+ .await+ .context("failed to reach the ECS task metadata endpoint")?+ .json()+ .await+ .context("failed to parse ECS task metadata")?;+ let ip = task["Containers"]+ .as_array()+ .into_iter()+ .flatten()+ .flat_map(|container| container["Networks"].as_array().into_iter().flatten())+ .flat_map(|network| network["IPv4Addresses"].as_array().into_iter().flatten())+ .find_map(serde_json::Value::as_str)+ .context("ECS task metadata carries no private IPv4 address")?;+ Ok(Some(agent_session::domain::model::ReplicaAddress::new(+ format!("http://{ip}:{port}"),+ )))+}+#[tokio::main]async fn main() -> anyhow::Result<()> {let entrypoint = MacroEntrypoint::default().init();@@ -177,6 +237,18 @@ async fn run() -> anyhow::Result<()> {// log and pushes each frame at the channel's participants so a viewer sees// it happen.let session_repo = PgAgentSessionRepo::new(pool.clone());+ // One lease identity for the whole process: commands forward to an+ // address, and both attach-capable service instances below live at this+ // one, so they share the id. The address is this task's own private URL,+ // discovered from ECS metadata; a deployment that cannot learn it must+ // not scale, so discovery failing anywhere but local refuses to boot.+ let replica = ReplicaId::mint();+ let replica_address = discover_replica_address(config.environment, config.port).await?;+ tracing::info!(+ %replica,+ address = ?replica_address.as_ref().map(ReplicaAddress::as_str),+ "harness replica identity"+ );let sessions = AgentSessionServiceImpl::new(session_repo.clone(),FoldedMessageService::new(session_repo.clone()),@@ -185,6 +257,7 @@ async fn run() -> anyhow::Result<()> {session_repo.clone(),),)+ .with_replica(replica).with_name_generator(HaikuAgentSessionNameGenerator::new(ai_usage::pg_recorder(pool.clone(),)));@@ -260,15 +333,13 @@ async fn run() -> anyhow::Result<()> {};// The sandbox provider serves every bot but the in-memory one, which the// router pulls out by bot id before the provider ever sees it.- let sandbox_and_inmem = RoutedContainers::new(- sandbox,- inmem,- AgentSessionServiceImpl::new(- session_repo.clone(),- FoldedMessageService::new(session_repo.clone()),- NoOpRealtime,- ),- );+ let inmem_sessions = AgentSessionServiceImpl::new(+ session_repo.clone(),+ FoldedMessageService::new(session_repo.clone()),+ NoOpRealtime,+ )+ .with_replica(replica);+ let sandbox_and_inmem = RoutedContainers::new(sandbox, inmem, inmem_sessions);// Cursor sessions run on their owner's own Cursor account, so there is no// deployment-wide key to arm this with: the manager reads each session@@ -406,6 +477,8 @@ async fn run() -> anyhow::Result<()> {prompt_context,prompt_composer,EgressProvisioner::new(Arc::clone(&mcp_connections), config.egress_base_url.clone()),+ HttpCommandForwarder::new(config.internal_api_key.clone())+ .map_err(|error| anyhow::anyhow!("failed to build the command forwarder: {error}"))?,defaults,));@@ -446,6 +519,10 @@ async fn run() -> anyhow::Result<()> {let gateway_state = RuntimeGatewayState::new(runtimes,bots_directory,+ MacroAuthorizationState::new(Arc::new(authorization_service.clone())),+ );+ let forward_state = ForwardGatewayState::new(+ harness.clone(),MacroAuthorizationState::new(Arc::new(authorization_service)),);let http_port = config.port;@@ -455,6 +532,7 @@ async fn run() -> anyhow::Result<()> {control_state,create_state,gateway_state,+ forward_state,http_port,shutdown_signal(),)@@ -464,6 +542,27 @@ async fn run() -> anyhow::Result<()> {}});+ // The session lease's liveness signal: while this beats, this process's+ // claims are held; when it stops - crash or shutdown - they go stale+ // within REPLICA_STALE_AFTER and any successor can claim. Graceful stops+ // release each claim eagerly in the actor teardown path; this loop is+ // what covers the ungraceful ones.+ let heartbeat_repo = session_repo.clone();+ let heartbeat_address = replica_address.clone();+ let heartbeat = tokio::spawn(async move {+ let mut ticker =+ tokio::time::interval(agent_session::domain::ports::REPLICA_HEARTBEAT_INTERVAL);+ loop {+ ticker.tick().await;+ if let Err(error) = heartbeat_repo+ .heartbeat(replica, heartbeat_address.as_ref())+ .await+ {+ tracing::warn!(error = ?error, %replica, "failed to heartbeat harness replica");+ }+ }+ });+// Keep trigger generation in this deployment while retaining Kafka as the// boundary between channel events and harness commands.let mut trigger = tokio::spawn(trigger::supervise(@@ -679,6 +778,7 @@ async fn run() -> anyhow::Result<()> {http.abort();trigger.abort();egress_http.abort();+ heartbeat.abort();let stop_failures = container_shutdown.shutdown_all().await;if stop_failures > 0 {tracing::error!(stop_failures, "some sandboxes failed to stop");