Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(agent_harness): horizontal harness — rolling deploys, fenced session lease, command forwarding (#6032)

Wolf Mermelstein7d07f6d4f03d
diff --git a/.sqlx/query-18fd730ea7a2734ac0d674edec980025a77971dac52498a2adb9470e6dc73787.json b/.sqlx/query-18fd730ea7a2734ac0d674edec980025a77971dac52498a2adb9470e6dc73787.json
new file mode 100644
index 0000000000..34ca00b9bf
--- /dev/null
+++ b/.sqlx/query-18fd730ea7a2734ac0d674edec980025a77971dac52498a2adb9470e6dc73787.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content)\n SELECT $1, $2, $3, $4, $5\n WHERE EXISTS (\n SELECT 1 FROM agent_session\n WHERE id = $2 AND manager_replica_id = $6 AND manager_fence = $7\n )\n RETURNING created_at\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "created_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Text",
+ "Text",
+ "Jsonb",
+ "Uuid",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "18fd730ea7a2734ac0d674edec980025a77971dac52498a2adb9470e6dc73787"
+}
diff --git a/.sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.json b/.sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.json
new file mode 100644
index 0000000000..6dc5347586
--- /dev/null
+++ b/.sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.json
@@ -0,0 +1,24 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH replica AS (\n INSERT INTO harness_replica (id, last_heartbeat_at)\n VALUES ($2, now())\n ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()\n )\n UPDATE agent_session\n SET manager_replica_id = $2,\n manager_fence = manager_fence + 1,\n modified_at = now()\n WHERE id = $1\n AND (\n manager_replica_id IS NULL\n OR manager_replica_id = $2\n OR NOT EXISTS (\n SELECT 1 FROM harness_replica live\n WHERE live.id = agent_session.manager_replica_id\n AND live.last_heartbeat_at > now() - make_interval(secs => $3)\n )\n )\n RETURNING manager_fence\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "manager_fence",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Float8"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd"
+}
diff --git a/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.json b/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.json
new file mode 100644
index 0000000000..e0fc201db3
--- /dev/null
+++ b/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.json
@@ -0,0 +1,29 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT live.id AS replica_id, live.address\n FROM agent_session\n JOIN harness_replica live ON live.id = agent_session.manager_replica_id\n WHERE agent_session.id = $1\n AND live.last_heartbeat_at > now() - make_interval(secs => $2)\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "replica_id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "address",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Float8"
+ ]
+ },
+ "nullable": [
+ false,
+ true
+ ]
+ },
+ "hash": "261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52"
+}
diff --git a/.sqlx/query-521b591873e9fdbaa3310380723aafad9da0742692aec0122c5c922804c5f795.json b/.sqlx/query-521b591873e9fdbaa3310380723aafad9da0742692aec0122c5c922804c5f795.json
new file mode 100644
index 0000000000..3e2d2c8469
--- /dev/null
+++ b/.sqlx/query-521b591873e9fdbaa3310380723aafad9da0742692aec0122c5c922804c5f795.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE agent_session\n SET manager_replica_id = NULL,\n modified_at = now()\n WHERE id = $1 AND manager_replica_id = $2 AND manager_fence = $3\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "521b591873e9fdbaa3310380723aafad9da0742692aec0122c5c922804c5f795"
+}
diff --git a/.sqlx/query-60620ad0a9e7fd8a9c8c61c331636e18332c96647f0b87bc29067d41104dd057.json b/.sqlx/query-60620ad0a9e7fd8a9c8c61c331636e18332c96647f0b87bc29067d41104dd057.json
new file mode 100644
index 0000000000..f2674f6222
--- /dev/null
+++ b/.sqlx/query-60620ad0a9e7fd8a9c8c61c331636e18332c96647f0b87bc29067d41104dd057.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO harness_replica (id, last_heartbeat_at, address)\n VALUES ($1, now(), $2)\n ON CONFLICT (id) DO UPDATE\n SET last_heartbeat_at = now(),\n address = COALESCE(EXCLUDED.address, harness_replica.address)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "60620ad0a9e7fd8a9c8c61c331636e18332c96647f0b87bc29067d41104dd057"
+}
diff --git a/.sqlx/query-8b9476b82011c40253d7dd36078e92e9fc4e0e295c9031b76879a8b472bb2aa5.json b/.sqlx/query-8b9476b82011c40253d7dd36078e92e9fc4e0e295c9031b76879a8b472bb2aa5.json
new file mode 100644
index 0000000000..248f9aaf6c
--- /dev/null
+++ b/.sqlx/query-8b9476b82011c40253d7dd36078e92e9fc4e0e295c9031b76879a8b472bb2aa5.json
@@ -0,0 +1,12 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM harness_replica WHERE last_heartbeat_at < now() - interval '7 days'",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": []
+ },
+ "hash": "8b9476b82011c40253d7dd36078e92e9fc4e0e295c9031b76879a8b472bb2aa5"
+}
diff --git a/.sqlx/query-a6d600a2285024d53f613e78fac8e24c64a44430b34b3f2f82edd6426ec26751.json b/.sqlx/query-a6d600a2285024d53f613e78fac8e24c64a44430b34b3f2f82edd6426ec26751.json
new file mode 100644
index 0000000000..9845f3c170
--- /dev/null
+++ b/.sqlx/query-a6d600a2285024d53f613e78fac8e24c64a44430b34b3f2f82edd6426ec26751.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT manager_replica_id FROM agent_session WHERE id = $1",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "manager_replica_id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ true
+ ]
+ },
+ "hash": "a6d600a2285024d53f613e78fac8e24c64a44430b34b3f2f82edd6426ec26751"
+}
diff --git a/Cargo.lock b/Cargo.lock
index 066eb86354..4088db34ff 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -324,6 +324,7 @@ dependencies = [
"notification",
"pipedream_mcp",
"rdkafka",
+ "reqwest 0.13.4",
"secretsmanager_client",
"serde",
"serde_json",
diff --git a/crates/agent_harness/src/domain/error.rs b/crates/agent_harness/src/domain/error.rs
index 7cc90717a3..73fc52d707 100644
--- a/crates/agent_harness/src/domain/error.rs
+++ b/crates/agent_harness/src/domain/error.rs
@@ -63,4 +63,12 @@ pub enum HarnessError {
/// A prompt could not be composed for the agent runtime.
#[error("failed to compose agent prompt: {0}")]
PromptComposition(rootcause::Report),
+ /// A live peer manages the session but published no address to forward
+ /// to. Transient by construction: only a replica from before addresses
+ /// existed, or one whose first heartbeat has not landed, has no address.
+ #[error("agent session {0} is managed by a live replica with no forwarding address")]
+ ManagerUnreachable(AgentSessionId),
+ /// Forwarding a command to the session's managing replica failed.
+ #[error("failed to forward an agent session command: {0}")]
+ Forward(rootcause::Report),
}
diff --git a/crates/agent_harness/src/domain/model.rs b/crates/agent_harness/src/domain/model.rs
index b8e9304d22..e1e8fc9892 100644
--- a/crates/agent_harness/src/domain/model.rs
+++ b/crates/agent_harness/src/domain/model.rs
@@ -9,7 +9,7 @@ use bot_id::BotId;
use macro_user_id::user_id::MacroUserIdStr;
use macro_uuid::Uuid;
/// Where a mention happened.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct MentionOrigin {
/// Channel the mentioning message was posted in.
pub channel_id: Uuid,
@@ -30,7 +30,7 @@ pub struct MentionOrigin {
/// [`agent_session::domain::ports::SessionOpener`] instead: they
/// need no provisioning, no announcement, and no first prompt, so they are
/// a plain create rather than a harness command.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct OpenSession {
/// The bot that was mentioned.
pub bot_id: BotId,
@@ -105,7 +105,7 @@ pub(crate) use agent_egress::domain::model::is_macro_staff;
/// Where a prompt came from, when it came from somewhere the session should
/// answer back into.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct AnnounceOrigin {
/// Channel the prompt was posted in.
pub channel_id: Uuid,
@@ -125,7 +125,7 @@ pub struct PriorChannelMessage {
}
/// Do something in a session that already exists.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct DeliverAction {
/// The id the action carries onto the wire, minted when it was accepted.
/// A reconnect-and-retry resends under the same id.
@@ -151,7 +151,7 @@ pub struct DeliverAction {
/// commands - whether to reconnect a dead session, whether to announce - are
/// properties of the action and its origin, not of the request that carried
/// it.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub enum HarnessCommand {
/// Open a new session.
Open(OpenSession),
diff --git a/crates/agent_harness/src/domain/ports.rs b/crates/agent_harness/src/domain/ports.rs
index 864f299fda..d2ba3f3afe 100644
--- a/crates/agent_harness/src/domain/ports.rs
+++ b/crates/agent_harness/src/domain/ports.rs
@@ -1,18 +1,56 @@
//! Outbound capabilities required by the harness domain.
use agent_session::domain::connection::RuntimeAttachment;
-use agent_session::domain::model::{AgentSessionId, SandboxSize};
+use agent_session::domain::model::{AgentSessionId, ReplicaAddress, SandboxSize};
use agent_session::domain::ports::AgentConnector;
use bot_id::BotId;
use macro_user_id::user_id::MacroUserIdStr;
-use super::error::Result;
+use super::error::{HarnessError, Result};
use super::model::{
- PriorChannelMessage, ProvisionedEgress, SandboxEgress, SessionAnnouncement, SpawnContainer,
+ HarnessCommand, PriorChannelMessage, ProvisionedEgress, SandboxEgress, SessionAnnouncement,
+ SpawnContainer,
};
use super::sandbox::SandboxResizeEffect;
+/// Delivers a session's command to the replica that manages its live actor.
+///
+/// The receiving side executes without re-resolving management (a forward is
+/// single-hop by contract, so two replicas with momentarily different views
+/// cannot bounce a command between each other). Success means the peer ran
+/// the command to completion - the response is the acknowledgment - so a
+/// caller that awaited a forward has the same guarantee as one that executed
+/// locally.
+pub trait CommandForwarder: Send + Sync + 'static {
+ /// Run `command` for `session` on the replica at `target`.
+ fn forward(
+ &self,
+ target: &ReplicaAddress,
+ session: AgentSessionId,
+ command: HarnessCommand,
+ ) -> impl Future<Output = Result<()>> + Send;
+}
+
+/// A forwarder for deployments with exactly one replica, where a live peer
+/// cannot exist: being asked to forward is itself the error, loudly, rather
+/// than a silent local fallback that would mask a mis-wiring.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct NoPeers;
+
+impl CommandForwarder for NoPeers {
+ async fn forward(
+ &self,
+ target: &ReplicaAddress,
+ session: AgentSessionId,
+ _command: HarnessCommand,
+ ) -> Result<()> {
+ Err(HarnessError::Forward(rootcause::report!(
+ "this deployment has no command forwarding, yet {target} manages session {session}"
+ )))
+ }
+}
+
#[cfg(test)]
mod test;
diff --git a/crates/agent_harness/src/domain/service.rs b/crates/agent_harness/src/domain/service.rs
index 77a613b121..61cc2c0275 100644
--- a/crates/agent_harness/src/domain/service.rs
+++ b/crates/agent_harness/src/domain/service.rs
@@ -6,6 +6,7 @@ use std::sync::Arc;
use agent_runtime_protocol::domain::action::{AgentAction, AgentActionId};
use agent_session::domain::connection::RuntimeAttachment;
use agent_session::domain::error::AgentSessionError;
+use agent_session::domain::model::SessionManagement;
use agent_session::domain::model::{
AgentSession, AgentSessionId, AuthorKind, CreateAgentSessionParams, MessageId, SandboxSize,
};
@@ -25,8 +26,8 @@ use crate::domain::model::{
OpenSession, SessionAnnouncement, SpawnContainer, is_macro_staff,
};
use crate::domain::ports::{
- AgentPromptComposer, ChannelPromptContext, ContainerManager, RuntimeConnections,
- SandboxEgressProvisioner, SessionAnnouncer,
+ AgentPromptComposer, ChannelPromptContext, CommandForwarder, ContainerManager,
+ RuntimeConnections, SandboxEgressProvisioner, SessionAnnouncer,
};
use crate::domain::sandbox::SandboxResizeEffect;
@@ -38,6 +39,37 @@ struct QueuedCommand {
/// The caller's span, carried across the queue so the work the worker does
/// on its own task still hangs off whatever triggered it.
span: tracing::Span,
+ /// Whether the worker resolves the session's managing replica before
+ /// executing. Commands admitted at an ingress route; a command received
+ /// *as* a forward executes here unconditionally, which is what makes
+ /// forwarding single-hop - two replicas with momentarily different lease
+ /// views cannot bounce a command between each other.
+ route: bool,
+}
+
+/// [`CommandForwarder`], object-safe.
+///
+/// Held erased inside the service so forwarding does not become an eighth
+/// type parameter on every impl block; the public port keeps its natural
+/// `impl Future` shape and this shim boxes at the one internal call site.
+trait ErasedForwarder: Send + Sync + 'static {
+ fn forward<'a>(
+ &'a self,
+ target: &'a agent_session::domain::model::ReplicaAddress,
+ session: AgentSessionId,
+ command: HarnessCommand,
+ ) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>>;
+}
+
+impl<F: CommandForwarder> ErasedForwarder for F {
+ fn forward<'a>(
+ &'a self,
+ target: &'a agent_session::domain::model::ReplicaAddress,
+ session: AgentSessionId,
+ command: HarnessCommand,
+ ) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {
+ Box::pin(CommandForwarder::forward(self, target, session, command))
+ }
}
struct AgentHarnessInner<
@@ -56,6 +88,7 @@ struct AgentHarnessInner<
prompt_context: PromptContext,
prompt_composer: PromptComposer,
egress: Egress,
+ forwarder: Box<dyn ErasedForwarder>,
defaults: HarnessDefaults,
}
@@ -115,6 +148,7 @@ where
prompt_context: PromptContext,
prompt_composer: PromptComposer,
egress: Egress,
+ forwarder: impl CommandForwarder,
defaults: impl Into<HarnessDefaults>,
) -> Self {
Self {
@@ -126,6 +160,7 @@ where
prompt_context,
prompt_composer,
egress,
+ forwarder: Box::new(forwarder),
defaults: defaults.into(),
}),
workers: Arc::new(DashMap::new()),
@@ -139,9 +174,34 @@ where
/// caller's span is captured: the returned future only awaits a oneshot, so
/// the span has to travel with the command to reach the work itself.
pub fn execute(
+ &self,
+ session_id: AgentSessionId,
+ command: HarnessCommand,
+ ) -> impl Future<Output = Result<()>> + Send + 'static {
+ self.enqueue(session_id, command, true)
+ }
+
+ /// [`execute`](Self::execute), without resolving which replica manages
+ /// the session first.
+ ///
+ /// The entry point for commands received *as* forwards: the sender
+ /// already resolved management to this replica, and re-resolving here is
+ /// what could bounce a command between two replicas whose lease views
+ /// momentarily differ. Forwarding is single-hop; this is the second hop's
+ /// half of that contract.
+ pub fn execute_here(
+ &self,
+ session_id: AgentSessionId,
+ command: HarnessCommand,
+ ) -> impl Future<Output = Result<()>> + Send + 'static {
+ self.enqueue(session_id, command, false)
+ }
+
+ fn enqueue(
&self,
session_id: AgentSessionId,
mut command: HarnessCommand,
+ route: bool,
) -> impl Future<Output = Result<()>> + Send + 'static {
let caller = tracing::Span::current();
let result = loop {
@@ -151,6 +211,7 @@ where
command,
completed,
span: caller.clone(),
+ route,
};
match commands.send(queued) {
@@ -238,6 +299,46 @@ where
}
}
+/// The receiving half of command forwarding: what the internal forward route
+/// calls, implemented by the harness as [`AgentHarnessService::execute_here`].
+pub trait ForwardedCommands: Send + Sync + 'static {
+ /// Run a command a peer already routed to this replica.
+ fn execute_forwarded(
+ &self,
+ session_id: AgentSessionId,
+ command: HarnessCommand,
+ ) -> impl Future<Output = Result<()>> + Send;
+}
+
+impl<Sessions, Containers, Announcer, Runtimes, PromptContext, PromptComposer, Egress>
+ ForwardedCommands
+ for AgentHarnessService<
+ Sessions,
+ Containers,
+ Announcer,
+ Runtimes,
+ PromptContext,
+ PromptComposer,
+ Egress,
+ >
+where
+ Sessions: AgentSessionService,
+ Containers: ContainerManager,
+ Announcer: SessionAnnouncer,
+ Runtimes: RuntimeConnections,
+ PromptContext: ChannelPromptContext,
+ PromptComposer: AgentPromptComposer,
+ Egress: SandboxEgressProvisioner,
+{
+ async fn execute_forwarded(
+ &self,
+ session_id: AgentSessionId,
+ command: HarnessCommand,
+ ) -> Result<()> {
+ self.execute_here(session_id, command).await
+ }
+}
+
/// The harness is what holds a session's live resources, so it is what the
/// control routes notify. Both operations go through the per-session queue, so
/// a teardown cannot land in the middle of an open and a model change cannot
@@ -541,6 +642,55 @@ where
PromptComposer: AgentPromptComposer,
Egress: SandboxEgressProvisioner,
{
+ /// Execute where the session's live actor is: locally when nobody (or
+ /// this replica) manages it, on the managing peer otherwise.
+ ///
+ /// A failed forward re-reads the lease once: the one legitimate reason a
+ /// live manager refuses its own session is that it died mid-flight, and
+ /// then its heartbeat going stale is what lets this replica take over. A
+ /// manager that is alive but unreachable stays an error - executing
+ /// locally anyway is how two actors end up on one session.
+ #[tracing::instrument(err, skip(self, command), fields(%session_id))]
+ async fn route_then_execute(
+ &self,
+ session_id: AgentSessionId,
+ command: HarnessCommand,
+ ) -> Result<()> {
+ // Open never routes: it is what creates the session row this routing
+ // would read, and a fresh id has no manager to defer to.
+ if matches!(command, HarnessCommand::Open(_)) {
+ return self.execute(session_id, command).await;
+ }
+ let manager = match self.sessions.management(session_id).await? {
+ SessionManagement::Unmanaged | SessionManagement::Ours => {
+ return self.execute(session_id, command).await;
+ }
+ SessionManagement::Peer(manager) => manager,
+ };
+ let Some(address) = manager.address else {
+ return Err(HarnessError::ManagerUnreachable(session_id));
+ };
+ tracing::info!(%session_id, peer = %manager.replica, "forwarding an agent session command");
+ match self
+ .forwarder
+ .forward(&address, session_id, command.clone())
+ .await
+ {
+ Ok(()) => Ok(()),
+ Err(forward_error) => match self.sessions.management(session_id).await? {
+ SessionManagement::Unmanaged | SessionManagement::Ours => {
+ tracing::warn!(
+ error = ?forward_error,
+ %session_id,
+ "the managing replica went stale mid-forward; executing locally"
+ );
+ self.execute(session_id, command).await
+ }
+ SessionManagement::Peer(_) => Err(forward_error),
+ },
+ }
+ }
+
async fn execute(&self, session_id: AgentSessionId, command: HarnessCommand) -> Result<()> {
match &command {
HarnessCommand::Open(open)
@@ -960,8 +1110,16 @@ async fn run_session_worker<
command,
completed,
span,
+ route,
} = queued;
- let result = inner.execute(session_id, command).instrument(span).await;
+ let result = if route {
+ inner
+ .route_then_execute(session_id, command)
+ .instrument(span)
+ .await
+ } else {
+ inner.execute(session_id, command).instrument(span).await
+ };
let _ = completed.send(result);
}
}
diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rs
index ec28d87289..291ba68017 100644
--- a/crates/agent_harness/src/domain/service/test.rs
+++ b/crates/agent_harness/src/domain/service/test.rs
@@ -35,7 +35,9 @@ use crate::domain::model::{
AgentKind, AnnounceOrigin, DeliverAction, HarnessCommand, HarnessDefaults, MentionOrigin,
OpenSession, PriorChannelMessage, SessionDefaults, SpawnContainer,
};
-use crate::domain::ports::{AgentPromptComposer, ChannelPromptContext, ContainerManager as _};
+use crate::domain::ports::{
+ AgentPromptComposer, ChannelPromptContext, ContainerManager as _, NoPeers,
+};
use crate::outbound::runtime_registry::RuntimeRegistry;
use crate::testing::helpers::agent::FakeAgent;
use crate::testing::helpers::announcer::AnnouncerMock;
@@ -210,6 +212,7 @@ fn harness_with_edges(
prompt_context,
prompt_composer,
EgressProvisionerMock::new(),
+ NoPeers,
SessionDefaults {
bot_id: BotId::TEST_A,
model: "claude".to_owned(),
@@ -1416,6 +1419,7 @@ async fn a_managed_session_opens_as_the_managed_default_bot() {
PromptContextMock::default(),
PromptComposerMock::default(),
EgressProvisionerMock::new(),
+ NoPeers,
HarnessDefaults::new(SessionDefaults {
bot_id: BotId::TEST_A,
model: "claude".to_owned(),
@@ -1801,3 +1805,157 @@ async fn set_sandbox_size_unsupported_does_not_persist() {
);
assert_eq!(containers.resumed(), 0);
}
+
+/// A [`CommandForwarder`] that records its calls and reports success.
+#[derive(Clone, Default)]
+struct RecordingForwarder {
+ calls: Arc<Mutex<Vec<(String, AgentSessionId)>>>,
+}
+
+impl crate::domain::ports::CommandForwarder for RecordingForwarder {
+ async fn forward(
+ &self,
+ target: &agent_session::domain::model::ReplicaAddress,
+ session: AgentSessionId,
+ _command: HarnessCommand,
+ ) -> crate::domain::error::Result<()> {
+ self.calls
+ .lock()
+ .unwrap()
+ .push((target.as_str().to_owned(), session));
+ Ok(())
+ }
+}
+
+/// A forwarder standing in for a peer that died mid-forward: the call fails,
+/// and by the time the caller re-reads the lease the peer's claim is gone.
+#[derive(Clone)]
+struct DyingPeerForwarder {
+ repo: InMemoryAgentSessionRepo,
+ claim: agent_session::domain::model::SessionClaim,
+}
+
+impl crate::domain::ports::CommandForwarder for DyingPeerForwarder {
+ async fn forward(
+ &self,
+ _target: &agent_session::domain::model::ReplicaAddress,
+ _session: AgentSessionId,
+ _command: HarnessCommand,
+ ) -> crate::domain::error::Result<()> {
+ use agent_session::domain::ports::SessionOwnership as _;
+ self.repo.release(&self.claim).await.expect("peer releases");
+ Err(HarnessError::Forward(rootcause::report!(
+ "the peer went away"
+ )))
+ }
+}
+
+/// Claim a session for a fabricated peer replica that publishes an address.
+async fn claim_as_peer(
+ repo: &InMemoryAgentSessionRepo,
+ session: AgentSessionId,
+ address: &str,
+) -> agent_session::domain::model::SessionClaim {
+ use agent_session::domain::model::{ClaimOutcome, ReplicaAddress, ReplicaId};
+ use agent_session::domain::ports::SessionOwnership as _;
+ let peer = ReplicaId::mint();
+ repo.heartbeat(peer, Some(&ReplicaAddress::new(address)))
+ .await
+ .expect("peer heartbeats");
+ match repo.claim(session, peer).await.expect("peer claims") {
+ ClaimOutcome::Claimed(claim) => claim,
+ ClaimOutcome::ManagedElsewhere(_) => panic!("nobody else should hold the test session"),
+ }
+}
+
+/// A command for a session a live peer manages goes to the peer's address
+/// and never executes here: the session outlives a Delete this replica would
+/// otherwise have applied.
+#[tokio::test]
+async fn commands_for_a_peer_managed_session_forward_to_its_address() {
+ let repo = InMemoryAgentSessionRepo::new();
+ let session = agent_session::testing::test_agent_session(AgentSessionId::new());
+ let id = session.id;
+ repo.insert_session(session);
+ let _claim = claim_as_peer(&repo, id, "http://10.0.0.7:8100").await;
+ let forwarder = RecordingForwarder::default();
+ let service = AgentHarnessService::new(
+ AgentSessionServiceImpl::new(
+ repo.clone(),
+ FoldedMessageService::new(repo.clone()),
+ NoOpRealtime,
+ ),
+ MockContainerManager::new(),
+ AnnouncerMock::new(),
+ RuntimeRegistry::<ContainerSender>::new(),
+ PromptContextMock::default(),
+ PromptComposerMock::default(),
+ EgressProvisionerMock::new(),
+ forwarder.clone(),
+ SessionDefaults {
+ bot_id: BotId::TEST_A,
+ model: "claude".to_owned(),
+ harness: "opencode".to_owned(),
+ repo_url: "https://github.com/macro-inc/macro".to_owned(),
+ },
+ );
+
+ service
+ .execute(id, HarnessCommand::Delete)
+ .await
+ .expect("the forwarded command succeeds");
+
+ assert_eq!(
+ forwarder.calls.lock().unwrap().clone(),
+ vec![("http://10.0.0.7:8100".to_owned(), id)]
+ );
+ assert!(
+ repo.get(id).await.is_ok(),
+ "the delete ran on the peer, not here"
+ );
+}
+
+/// A forward that fails against a peer whose lease is gone by the re-read
+/// falls back to executing locally, so a peer crash costs one retry rather
+/// than the command.
+#[tokio::test]
+async fn a_dead_peers_command_falls_back_to_local_execution() {
+ let repo = InMemoryAgentSessionRepo::new();
+ let session = agent_session::testing::test_agent_session(AgentSessionId::new());
+ let id = session.id;
+ repo.insert_session(session);
+ let claim = claim_as_peer(&repo, id, "http://10.0.0.7:8100").await;
+ let service = AgentHarnessService::new(
+ AgentSessionServiceImpl::new(
+ repo.clone(),
+ FoldedMessageService::new(repo.clone()),
+ NoOpRealtime,
+ ),
+ MockContainerManager::new(),
+ AnnouncerMock::new(),
+ RuntimeRegistry::<ContainerSender>::new(),
+ PromptContextMock::default(),
+ PromptComposerMock::default(),
+ EgressProvisionerMock::new(),
+ DyingPeerForwarder {
+ repo: repo.clone(),
+ claim,
+ },
+ SessionDefaults {
+ bot_id: BotId::TEST_A,
+ model: "claude".to_owned(),
+ harness: "opencode".to_owned(),
+ repo_url: "https://github.com/macro-inc/macro".to_owned(),
+ },
+ );
+
+ service
+ .execute(id, HarnessCommand::Delete)
+ .await
+ .expect("the fallback executes locally");
+
+ assert!(
+ repo.get(id).await.is_err(),
+ "the delete ran here once the peer was gone"
+ );
+}
diff --git a/crates/agent_harness/src/inbound/forward.rs b/crates/agent_harness/src/inbound/forward.rs
new file mode 100644
index 0000000000..6e428695f0
--- /dev/null
+++ b/crates/agent_harness/src/inbound/forward.rs
@@ -0,0 +1,105 @@
+//! The receiving half of replica-to-replica command forwarding.
+//!
+//! A peer that consumed a command for a session this replica manages POSTs it
+//! here. Internal-only by construction: the caller is another replica of this
+//! same deployment, authenticated with the shared internal key, never a user.
+//! The handler executes without re-resolving management - forwarding is
+//! single-hop, and this endpoint is the second hop honoring that contract.
+
+use std::sync::Arc;
+
+use agent_session::domain::model::AgentSessionId;
+use axum::Router;
+use axum::extract::{FromRef, Json, Path, State};
+use axum::http::StatusCode;
+use axum::response::{IntoResponse, Response};
+use axum::routing::post;
+use macro_authorization::{
+ InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService, MacroAuthorizationState,
+};
+use macro_uuid::Uuid;
+
+use crate::domain::error::HarnessError;
+use crate::domain::model::HarnessCommand;
+use crate::domain::service::ForwardedCommands;
+
+/// State for the command-forwarding route.
+pub struct ForwardGatewayState<Harness, Auth> {
+ harness: Arc<Harness>,
+ authorization_state: MacroAuthorizationState<Auth>,
+}
+
+impl<Harness, Auth> ForwardGatewayState<Harness, Auth> {
+ /// Create forwarding state.
+ pub fn new(harness: Arc<Harness>, authorization_state: MacroAuthorizationState<Auth>) -> Self {
+ Self {
+ harness,
+ authorization_state,
+ }
+ }
+}
+
+// Manual Clone impl: everything is behind an Arc.
+impl<Harness, Auth> Clone for ForwardGatewayState<Harness, Auth> {
+ fn clone(&self) -> Self {
+ Self {
+ harness: Arc::clone(&self.harness),
+ authorization_state: self.authorization_state.clone(),
+ }
+ }
+}
+
+impl<Harness, Auth> FromRef<ForwardGatewayState<Harness, Auth>> for MacroAuthorizationState<Auth> {
+ fn from_ref(state: &ForwardGatewayState<Harness, Auth>) -> Self {
+ state.authorization_state.clone()
+ }
+}
+
+/// Build the router serving `POST /agent-sessions/{id}/command`. Mount it
+/// under `/internal`.
+pub fn forward_router<Harness, Auth, S>(state: ForwardGatewayState<Harness, Auth>) -> Router<S>
+where
+ Harness: ForwardedCommands,
+ Auth: MacroAuthorizationService,
+ S: Clone + Send + Sync + 'static,
+{
+ Router::new()
+ .route(
+ "/agent-sessions/{session_id}/command",
+ post(forward_handler::<Harness, Auth>),
+ )
+ .with_state(state)
+}
+
+/// Execute a command a peer replica forwarded to this one.
+async fn forward_handler<Harness, Auth>(
+ State(state): State<ForwardGatewayState<Harness, Auth>>,
+ _caller: MacroAuthorizationExtractor<Auth, InternalOnly>,
+ Path(session_id): Path<Uuid>,
+ Json(command): Json<HarnessCommand>,
+) -> Response
+where
+ Harness: ForwardedCommands,
+ Auth: MacroAuthorizationService,
+{
+ let session_id = AgentSessionId::new_from_uuid(session_id);
+ match state.harness.execute_forwarded(session_id, command).await {
+ Ok(()) => StatusCode::NO_CONTENT.into_response(),
+ // Surfaced with its own status so the sender can tell "the actor is
+ // not here after all" from an execution failure: the sender's
+ // fallback re-reads the lease on any error, but a 409 is the signal
+ // that its view was stale rather than that the command is bad.
+ Err(HarnessError::Disconnected(id))
+ | Err(HarnessError::Session(
+ agent_session::domain::error::AgentSessionError::Disconnected(id),
+ )) => (
+ StatusCode::CONFLICT,
+ format!("session {id} is not attached here"),
+ )
+ .into_response(),
+ Err(error) => {
+ tracing::error!(error = ?error, %session_id, "a forwarded command failed");
+ (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response()
+ }
+ }
+}
diff --git a/crates/agent_harness/src/inbound/mod.rs b/crates/agent_harness/src/inbound/mod.rs
index 856caaa13c..aeac45aac8 100644
--- a/crates/agent_harness/src/inbound/mod.rs
+++ b/crates/agent_harness/src/inbound/mod.rs
@@ -3,5 +3,6 @@
//! Thin by construction - decode the broker message, ask the domain whether it
//! is addressed to us, hand one value inward. No policy lives here.
+pub mod forward;
pub mod kafka;
pub mod runtime_gateway;
diff --git a/crates/agent_harness/src/outbound/forward.rs b/crates/agent_harness/src/outbound/forward.rs
new file mode 100644
index 0000000000..87fb4c6d55
--- /dev/null
+++ b/crates/agent_harness/src/outbound/forward.rs
@@ -0,0 +1,83 @@
+//! HTTP implementation of [`CommandForwarder`]: one authenticated POST to the
+//! managing replica's internal forward route, awaited for its result.
+//!
+//! Direct replica-to-replica, not through the load balancer: the target
+//! address is the peer's own private URL as published with its heartbeat, so
+//! the command lands on exactly the process holding the session's actor.
+
+use agent_session::domain::model::{AgentSessionId, ReplicaAddress};
+use reqwest::StatusCode;
+
+use crate::domain::error::{HarnessError, Result};
+use crate::domain::model::HarnessCommand;
+use crate::domain::ports::CommandForwarder;
+
+/// Header carrying the deployment's shared internal key, as
+/// `macro_authorization`'s internal extractor reads it.
+const INTERNAL_API_KEY_HEADER: &str = macro_authorization::INTERNAL_API_KEY_HEADER;
+
+/// Forwards commands to peer replicas over their internal HTTP surface.
+#[derive(Clone)]
+pub struct HttpCommandForwarder {
+ client: reqwest::Client,
+ internal_api_key: String,
+}
+
+impl HttpCommandForwarder {
+ /// A forwarder authenticating with the deployment's internal key.
+ ///
+ /// The timeout bounds the whole forwarded execution, not just the dial:
+ /// the peer replies only once the command ran, and a Deliver can sit
+ /// behind a sandbox resume, so this is generous rather than snappy.
+ pub fn new(internal_api_key: String) -> Result<Self> {
+ let client = reqwest::Client::builder()
+ .connect_timeout(std::time::Duration::from_secs(2))
+ .timeout(std::time::Duration::from_secs(120))
+ .build()
+ .map_err(|error| HarnessError::Forward(rootcause::report!(error).into_dynamic()))?;
+ Ok(Self {
+ client,
+ internal_api_key,
+ })
+ }
+}
+
+impl CommandForwarder for HttpCommandForwarder {
+ #[tracing::instrument(err, skip(self, command), fields(%target, %session))]
+ async fn forward(
+ &self,
+ target: &ReplicaAddress,
+ session: AgentSessionId,
+ command: HarnessCommand,
+ ) -> Result<()> {
+ let url = format!(
+ "{}/internal/agent-sessions/{}/command",
+ target.as_str().trim_end_matches('/'),
+ session
+ );
+ let response = self
+ .client
+ .post(url)
+ .header(INTERNAL_API_KEY_HEADER, &self.internal_api_key)
+ .json(&command)
+ .send()
+ .await
+ .map_err(|error| HarnessError::Forward(rootcause::report!(error).into_dynamic()))?;
+ let status = response.status();
+ if status.is_success() {
+ return Ok(());
+ }
+ let body = response.text().await.unwrap_or_default();
+ // 409 is the peer saying "not attached here after all" - the caller's
+ // stale-view fallback keys off any error, but keep the distinction
+ // visible in the report for whoever reads the logs.
+ let refused = if status == StatusCode::CONFLICT {
+ "the peer refused the session"
+ } else {
+ "the peer failed the command"
+ };
+ Err(HarnessError::Forward(rootcause::report!(
+ "{refused}: {status}: {body}"
+ )))
+ }
+}
diff --git a/crates/agent_harness/src/outbound/mod.rs b/crates/agent_harness/src/outbound/mod.rs
index 7fb25fdddb..6f7678e264 100644
--- a/crates/agent_harness/src/outbound/mod.rs
+++ b/crates/agent_harness/src/outbound/mod.rs
@@ -8,6 +8,7 @@ pub mod containers;
pub mod cursor;
pub mod daytona;
pub mod egress;
+pub mod forward;
pub mod local;
pub(crate) mod managed_containers;
pub mod namespace;
diff --git a/crates/agent_session/src/domain/error.rs b/crates/agent_session/src/domain/error.rs
index 83f4afd230..3f79166aa6 100644
--- a/crates/agent_session/src/domain/error.rs
+++ b/crates/agent_session/src/domain/error.rs
@@ -8,6 +8,10 @@ pub type Result<T, E = AgentSessionError> = std::result::Result<T, E>;
pub enum AgentSessionError {
#[error("agent session {0} already has an active transport")]
AlreadyConnected(AgentSessionId),
+ #[error("agent session {0} is managed by another live replica")]
+ ManagedElsewhere(AgentSessionId),
+ #[error("agent session {0} write was fenced out: another replica claimed the session")]
+ FencedOut(AgentSessionId),
#[error("acp handshake failed: {0}")]
Handshake(String),
#[error("agent session {0} is no longer connected")]
diff --git a/crates/agent_session/src/domain/model.rs b/crates/agent_session/src/domain/model.rs
index 28cd843c8e..a3021121e4 100644
--- a/crates/agent_session/src/domain/model.rs
+++ b/crates/agent_session/src/domain/model.rs
@@ -15,6 +15,123 @@ pub use agent_fold::domain::model::{
Author, AuthorKind, FoldEvent, MessageId, OwnedFoldEvent, TurnId,
};
+/// Identity of one harness participant, minted fresh at construction.
+///
+/// A restarted process is a new replica: whatever the old identity claimed is
+/// released by its heartbeat going stale, never inherited.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct ReplicaId(Uuid);
+
+impl ReplicaId {
+ /// Mint a fresh replica identity.
+ #[must_use]
+ pub fn mint() -> Self {
+ Self(Uuid::new_v4())
+ }
+
+ /// The raw uuid, for persistence.
+ #[must_use]
+ pub fn as_uuid(&self) -> Uuid {
+ self.0
+ }
+
+ /// Rebuild an identity from its persisted uuid.
+ #[must_use]
+ pub fn from_uuid(id: Uuid) -> Self {
+ Self(id)
+ }
+}
+
+impl std::fmt::Display for ReplicaId {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+/// A replica's own base URL, as peers should dial it for command forwarding.
+///
+/// Private-network address discovered by the replica itself at boot (the ECS
+/// task metadata endpoint in deployments), published with its heartbeat.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ReplicaAddress(String);
+
+impl ReplicaAddress {
+ /// Wrap a base URL, e.g. `http://10.0.1.7:8100`.
+ #[must_use]
+ pub fn new(address: impl Into<String>) -> Self {
+ Self(address.into())
+ }
+
+ /// The base URL as a string.
+ #[must_use]
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+impl std::fmt::Display for ReplicaAddress {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+/// The live manager of a session, as read from the lease.
+#[derive(Debug, Clone)]
+pub struct SessionManager {
+ /// The replica holding the claim.
+ pub replica: ReplicaId,
+ /// Where to forward its commands, when the replica has published one.
+ /// `None` means the manager is live but unreachable - hold the error
+ /// rather than execute somewhere the actor is not.
+ pub address: Option<ReplicaAddress>,
+}
+
+/// Where a session's live actor runs, from one service instance's viewpoint.
+#[derive(Debug, Clone)]
+pub enum SessionManagement {
+ /// No live replica manages the session; this instance may claim it by
+ /// attaching, so commands execute locally.
+ Unmanaged,
+ /// This instance's replica manages it; commands execute locally.
+ Ours,
+ /// A live peer manages it; commands belong at its address.
+ Peer(SessionManager),
+}
+
+/// A session's takeover counter, bumped by every successful claim.
+///
+/// Carried by the claim holder into each live-actor write; the store rejects
+/// writes whose fence has been superseded, so a stale holder is neutralized
+/// by the same statement that would have written (a fencing token).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct ManagerFence(pub i64);
+
+/// Proof that this replica claimed a session's live management.
+///
+/// Obtained only from [`SessionOwnership::claim`](super::ports::SessionOwnership::claim);
+/// holding one is what entitles an actor to attach and write the session's
+/// log under its fence.
+#[derive(Debug, Clone, Copy)]
+pub struct SessionClaim {
+ /// The claimed session.
+ pub session: AgentSessionId,
+ /// The replica holding the claim.
+ pub replica: ReplicaId,
+ /// The fence this claim writes under.
+ pub fence: ManagerFence,
+}
+
+/// What claiming a session yielded.
+#[derive(Debug, Clone, Copy)]
+pub enum ClaimOutcome {
+ /// This replica now manages the session.
+ Claimed(SessionClaim),
+ /// A replica with a fresh heartbeat already manages it. Until command
+ /// forwarding exists this surfaces as an error; with it, commands are
+ /// routed to the named replica instead.
+ ManagedElsewhere(ReplicaId),
+}
+
/// Display name assigned to a newly created agent session.
pub const DEFAULT_AGENT_SESSION_NAME: &str = "Agent Session";
diff --git a/crates/agent_session/src/domain/ports.rs b/crates/agent_session/src/domain/ports.rs
index 73eb13028f..7a1f321422 100644
--- a/crates/agent_session/src/domain/ports.rs
+++ b/crates/agent_session/src/domain/ports.rs
@@ -272,6 +272,62 @@ pub trait ExternalSessionRepo: Send + Sync + 'static {
fn delete(&self, id: AgentSessionId) -> impl Future<Output = Result<()>> + Send;
}
+/// How often a replica refreshes its heartbeat row.
+pub const REPLICA_HEARTBEAT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(10);
+
+/// How stale a replica's heartbeat may be before its claims are up for
+/// grabs. Three missed heartbeats: long enough that one slow write does not
+/// get a live replica's sessions stolen, short enough that a crashed
+/// replica's sessions resume on the next prompt rather than minutes later.
+pub const REPLICA_STALE_AFTER: std::time::Duration = std::time::Duration::from_secs(30);
+
+/// The session-management lease: which replica holds a session's live actor.
+///
+/// Implemented by the same store that persists the session log, never
+/// separately - the fence a claim carries is checked by that store's fenced
+/// appends, and minting fences in one place while checking them in another
+/// is the mis-wiring this coupling forbids.
+///
+/// Claiming is a single conditional update (compare-and-swap): it succeeds
+/// when the session is unmanaged, already ours, or held by a replica whose
+/// heartbeat has gone stale, and every success increments the session's
+/// fence. There are deliberately no explicit locks anywhere in the contract -
+/// see [`ManagerFence`](super::model::ManagerFence) for why a fence, not a
+/// lock, is what neutralizes a stale holder.
+pub trait SessionOwnership: Send + Sync + 'static {
+ /// Claim live management of a session for `replica`, registering the
+ /// replica's heartbeat as a side effect so a claim can never dangle on a
+ /// replica the store has not seen.
+ fn claim(
+ &self,
+ session: AgentSessionId,
+ replica: ReplicaId,
+ ) -> impl Future<Output = Result<ClaimOutcome>> + Send;
+
+ /// Release a claim this replica holds. Conditional on the claim's fence
+ /// still being current: releasing after having been superseded is a
+ /// no-op, never a theft of the successor's claim. A session already
+ /// released (or deleted) is in the asked-for state, so this succeeds.
+ fn release(&self, claim: &SessionClaim) -> impl Future<Output = Result<()>> + Send;
+
+ /// Refresh this replica's heartbeat, upserting its row and publishing
+ /// `address` - the base URL peers forward this replica's sessions'
+ /// commands to - when one is known.
+ fn heartbeat(
+ &self,
+ replica: ReplicaId,
+ address: Option<&ReplicaAddress>,
+ ) -> impl Future<Output = Result<()>> + Send;
+
+ /// The live manager of a session, if a replica with a fresh heartbeat
+ /// holds its lease. `None` covers both an unclaimed session and one whose
+ /// holder has gone stale - either way the session is claimable.
+ fn manager_of(
+ &self,
+ session: AgentSessionId,
+ ) -> impl Future<Output = Result<Option<SessionManager>>> + Send;
+}
+
#[cfg_attr(feature = "test-utils", mockall::automock)]
pub trait AgentSessionLogRepo: Send + Sync + 'static {
/// Append a log entry and project any system event onto the session status.
@@ -280,6 +336,21 @@ pub trait AgentSessionLogRepo: Send + Sync + 'static {
log: AgentSessionLog,
) -> impl Future<Output = Result<StoredAgentSessionLog>> + Send;
+ /// [`create`](Self::create), conditioned on `claim` still holding the
+ /// session's current fence.
+ ///
+ /// This is the write half of the fencing contract: the check and the
+ /// append are one atomic statement, so a replica that stalled past its
+ /// heartbeat and was superseded cannot interleave frames no matter when
+ /// it wakes - its append matches nothing and fails with
+ /// [`FencedOut`](super::error::AgentSessionError::FencedOut), which the
+ /// actor treats as its cue to tear down.
+ fn create_fenced(
+ &self,
+ log: AgentSessionLog,
+ claim: &SessionClaim,
+ ) -> impl Future<Output = Result<StoredAgentSessionLog>> + Send;
+
/// List all log entries for a session, in chronological order.
///
/// Entries come back stamped with when the log recorded them: the frame
diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rs
index 00e8132b6a..6cec76b604 100644
--- a/crates/agent_session/src/domain/service.rs
+++ b/crates/agent_session/src/domain/service.rs
@@ -47,12 +47,13 @@ use super::connection::RuntimeAttachment;
use super::error::{AgentSessionError, Result};
use super::model::{
AgentSession, AgentSessionId, AgentSessionLog, AgentSessionRenamed, AuthorKind, ChannelSession,
- CreateAgentSessionParams, LogAppended, MAX_AGENT_SESSION_NAME_CHARS, Message, MessageId,
- SandboxSize, SessionLog, StoredAgentSessionLog,
+ ClaimOutcome, CreateAgentSessionParams, LogAppended, MAX_AGENT_SESSION_NAME_CHARS, Message,
+ MessageId, ReplicaId, SandboxSize, SessionClaim, SessionLog, SessionManagement,
+ StoredAgentSessionLog,
};
use super::ports::{
AgentConnector, AgentSessionLogRepo, AgentSessionLogWriter, AgentSessionNameGenerator,
- AgentSessionRealtime, AgentSessionRepo, NoOpAgentSessionNameGenerator,
+ AgentSessionRealtime, AgentSessionRepo, NoOpAgentSessionNameGenerator, SessionOwnership,
};
use super::session::actors::{SessionActor, SessionCommand, Stepped};
use super::session::{CloseReason, Input};
@@ -148,6 +149,14 @@ pub trait AgentSessionService: Send + Sync + 'static {
/// Persist that a session disconnected before a live actor could report it.
fn mark_disconnected(&self, id: AgentSessionId) -> impl Future<Output = Result<()>> + Send;
+ /// Where the session's live actor runs, from this instance's viewpoint:
+ /// unmanaged (claimable here), ours, or a live peer's - in which case
+ /// commands belong at the peer's address rather than in this process.
+ fn management(
+ &self,
+ id: AgentSessionId,
+ ) -> impl Future<Output = Result<SessionManagement>> + Send;
+
/// Attach a new transport to an existing persisted session.
///
/// The attachment carries the connection's handshake gate as well as the
@@ -230,6 +239,10 @@ pub struct AgentSessionServiceImpl<R, Folds, Rt, Namer = NoOpAgentSessionNameGen
realtime: Rt,
name_generator: Namer,
active: Arc<ActiveSessions>,
+ /// This service's identity in the session-management lease. Minted at
+ /// construction: a restarted process is a new replica, and its claims
+ /// are recovered by heartbeat staleness, never inherited.
+ replica: ReplicaId,
tasks: TaskTracker,
cancellation: CancellationToken,
lifecycle: Arc<Mutex<()>>,
@@ -249,6 +262,7 @@ impl<R, Folds, Rt> AgentSessionServiceImpl<R, Folds, Rt> {
realtime,
name_generator: NoOpAgentSessionNameGenerator,
active: Arc::new(DashMap::new()),
+ replica: ReplicaId::mint(),
tasks: TaskTracker::new(),
cancellation: CancellationToken::new(),
lifecycle: Arc::new(Mutex::new(())),
@@ -269,12 +283,33 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
realtime: self.realtime,
name_generator,
active: self.active,
+ replica: self.replica,
tasks: self.tasks,
cancellation: self.cancellation,
lifecycle: self.lifecycle,
}
}
+ /// This service's identity in the session-management lease, for the
+ /// composition root to heartbeat while the process lives.
+ #[must_use]
+ pub fn replica_id(&self) -> ReplicaId {
+ self.replica
+ }
+
+ /// Adopt a caller-minted replica identity, replacing the constructor's.
+ ///
+ /// For processes with more than one attach-capable service instance: the
+ /// lease's replica is the *process* (commands forward to an address, and
+ /// every instance in a process shares one), so the composition root mints
+ /// a single id and stamps it on each instance. The instances' session
+ /// sets are disjoint by construction, so they never contend for a lease.
+ #[must_use]
+ pub fn with_replica(mut self, replica: ReplicaId) -> Self {
+ self.replica = replica;
+ self
+ }
+
/// Stop active actors and wait for their tasks to release their transports.
pub async fn shutdown(&self) {
let lifecycle = self.lifecycle.lock().await;
@@ -322,9 +357,10 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
session: AgentSession,
attachment: RuntimeAttachment<Connector>,
reservation: AttachReservation,
+ claim: SessionClaim,
) -> Result<()>
where
- R: AgentSessionRepo + AgentSessionLogRepo + Clone,
+ R: AgentSessionRepo + AgentSessionLogRepo + SessionOwnership + Clone,
Rt: AgentSessionRealtime + Clone + Send + Sync + 'static,
Connector: AgentConnector,
{
@@ -348,8 +384,10 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
// rather than the bare repository - see module docs. Its fold starts
// empty and catches itself up on the stored log on the first frame,
// which costs an attach nothing until the session actually says
- // something.
- let logs = LiveSessionLogWriter::new(self.repo.clone(), self.realtime.clone());
+ // something. Fenced under the claim taken above: if another replica
+ // supersedes this one, the store rejects the next append and the
+ // actor tears down through its ordinary log-failure path.
+ let logs = LiveSessionLogWriter::fenced(self.repo.clone(), self.realtime.clone(), claim);
let actor = SessionActor::new(
id,
session.acp_session_id,
@@ -367,6 +405,8 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
marker,
stopped_tx,
self.cancellation.clone(),
+ self.repo.clone(),
+ claim,
)
.with_current_subscriber(),
);
@@ -476,7 +516,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
impl<R, Folds, Rt, Namer> AgentSessionService for AgentSessionServiceImpl<R, Folds, Rt, Namer>
where
- R: AgentSessionRepo + AgentSessionLogRepo + Clone,
+ R: AgentSessionRepo + AgentSessionLogRepo + SessionOwnership + Clone,
Folds: FoldedMessageRepo + Clone + Send + Sync + 'static,
Rt: AgentSessionRealtime + Clone + Send + Sync + 'static,
Namer: AgentSessionNameGenerator + Clone,
@@ -547,6 +587,14 @@ where
Ok(())
}
+ async fn management(&self, id: AgentSessionId) -> Result<SessionManagement> {
+ Ok(match self.repo.manager_of(id).await? {
+ None => SessionManagement::Unmanaged,
+ Some(manager) if manager.replica == self.replica => SessionManagement::Ours,
+ Some(manager) => SessionManagement::Peer(manager),
+ })
+ }
+
async fn mark_disconnected(&self, id: AgentSessionId) -> Result<()> {
let mut logs = LiveSessionLogWriter::new(self.repo.clone(), self.realtime.clone());
tokio::time::timeout(
@@ -571,10 +619,42 @@ where
where
Connector: AgentConnector,
{
+ // Reservation first: it is the in-process exclusion, so no sibling
+ // attach of this instance can race us to the claim below - which
+ // matters because every successful claim bumps the fence, and bumping
+ // it under a live actor of our own would fence that actor out.
let reservation = self.reserve_attach(id).await?;
- let session = self.repo.get(id).await?;
- self.activate_reserved(session, attachment, reservation)
- .await
+ let claim = match self.repo.claim(id, self.replica).await? {
+ ClaimOutcome::Claimed(claim) => claim,
+ ClaimOutcome::ManagedElsewhere(holder) => {
+ tracing::info!(%id, %holder, "agent session is managed by another live replica");
+ return Err(AgentSessionError::ManagedElsewhere(id));
+ }
+ };
+ let activated = async {
+ let session = self.repo.get(id).await?;
+ self.activate_reserved(session, attachment, reservation, claim)
+ .await
+ }
+ .await;
+ if let Err(error) = activated {
+ // The actor that would have released this claim never started;
+ // free it here so another replica is not left waiting out our
+ // heartbeat to resume the session.
+ self.repo
+ .release(&claim)
+ .await
+ .inspect_err(|release_error| {
+ tracing::error!(
+ error = ?release_error,
+ %id,
+ "failed to release an agent session claim after a failed attach"
+ );
+ })
+ .ok();
+ return Err(error);
+ }
+ Ok(())
}
async fn send_action(
@@ -740,6 +820,12 @@ pub struct LiveSessionLogWriter<R, Rt> {
repo: R,
realtime: Rt,
fold: Option<FoldMachineImpl>,
+ /// The management claim this writer appends under, when it has one. A
+ /// session actor always writes fenced; the unfenced constructor exists
+ /// for writers outside any live-management contest - `seed_jsonl`
+ /// replaying a recording, and `mark_disconnected` recording that a
+ /// runtime dropped before anything attached.
+ claim: Option<SessionClaim>,
}
impl<R, Rt> LiveSessionLogWriter<R, Rt> {
@@ -754,6 +840,18 @@ impl<R, Rt> LiveSessionLogWriter<R, Rt> {
repo,
realtime,
fold: None,
+ claim: None,
+ }
+ }
+
+ /// [`new`](Self::new), with every append conditioned on `claim` still
+ /// holding the session's current fence. What a live actor writes with.
+ pub fn fenced(repo: R, realtime: Rt, claim: SessionClaim) -> Self {
+ Self {
+ repo,
+ realtime,
+ fold: None,
+ claim: Some(claim),
}
}
}
@@ -783,7 +881,10 @@ where
// Durable first: projections are rebuildable, but a frame omitted from
// session history is not.
- let stored = AgentSessionLogRepo::create(&self.repo, log.clone()).await?;
+ let stored = match &self.claim {
+ Some(claim) => self.repo.create_fenced(log.clone(), claim).await?,
+ None => AgentSessionLogRepo::create(&self.repo, log.clone()).await?,
+ };
if let Some(fold) = &mut self.fold {
let _ = fold.push(log.clone());
@@ -967,16 +1068,20 @@ where
}
}
-/// Step the actor until its machine stops, then release the registry entry.
-async fn run_session<Connector, Logs>(
+/// Step the actor until its machine stops, then release the registry entry
+/// and the session's management claim.
+async fn run_session<Connector, Logs, Ownership>(
mut actor: SessionActor<Connector, Logs>,
active: std::sync::Weak<ActiveSessions>,
marker: Arc<()>,
stopped: watch::Sender<bool>,
cancellation: CancellationToken,
+ ownership: Ownership,
+ claim: SessionClaim,
) where
Connector: AgentConnector,
Logs: AgentSessionLogWriter + AgentSessionRepo,
+ Ownership: SessionOwnership,
{
loop {
let input = tokio::select! {
@@ -997,6 +1102,13 @@ async fn run_session<Connector, Logs>(
// Tear down the old transport before allowing another actor to attach.
drop(actor);
+ // Give the claim back before announcing the stop: fence-conditioned, so
+ // if a successor already took over this quietly does nothing. Releasing
+ // here rather than waiting for heartbeat staleness is what lets another
+ // replica resume this session immediately after a graceful stop.
+ if let Err(error) = ownership.release(&claim).await {
+ tracing::error!(error = ?error, %id, "failed to release an agent session claim");
+ }
let _ = stopped.send(true);
if let Some(active) = active.upgrade() {
active.remove_if(&id, |_, current| {
diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rs
index bc45b57572..e2ee17977f 100644
--- a/crates/agent_session/src/domain/service/test.rs
+++ b/crates/agent_session/src/domain/service/test.rs
@@ -1,6 +1,8 @@
use super::*;
use crate::PROTOCOL_VERSION;
-use crate::domain::model::{DEFAULT_AGENT_SESSION_NAME, Message, SessionBot};
+use crate::domain::model::{
+ DEFAULT_AGENT_SESSION_NAME, Message, ReplicaAddress, SessionBot, SessionManager,
+};
use crate::domain::ports::NoOpRealtime;
use crate::domain::session::HandshakeStatus;
use crate::testing::{InMemoryAgentSessionRepo, RecordingRealtime, test_agent_session};
@@ -162,6 +164,21 @@ impl AgentSessionLogWriter for BlockingPromptLogs {
}
}
+/// Claim a session's management for a freshly minted replica, as
+/// `attach_session` does before activating.
+async fn claim_for_test(repo: &InMemoryAgentSessionRepo, session: AgentSessionId) -> SessionClaim {
+ match repo
+ .claim(session, ReplicaId::mint())
+ .await
+ .expect("claim for test")
+ {
+ ClaimOutcome::Claimed(claim) => claim,
+ ClaimOutcome::ManagedElsewhere(holder) => {
+ panic!("test session is unexpectedly managed by {holder}")
+ }
+ }
+}
+
fn owner_access(session: AgentSessionId) -> EntityAccessReceipt<OwnerAccessLevel> {
EntityAccessReceipt::dangerously_assert_internal_user(
&session.as_uuid().to_string(),
@@ -400,7 +417,35 @@ impl AgentSessionRepo for BlockingPromptLogs {
}
}
+/// Pure delegation: the lease semantics under test live in the shared
+/// in-memory store, and this wrapper only intercepts log writes.
+impl SessionOwnership for BlockingPromptLogs {
+ async fn claim(&self, session: AgentSessionId, replica: ReplicaId) -> Result<ClaimOutcome> {
+ self.repo.claim(session, replica).await
+ }
+
+ async fn release(&self, claim: &SessionClaim) -> Result<()> {
+ self.repo.release(claim).await
+ }
+
+ async fn heartbeat(&self, replica: ReplicaId, address: Option<&ReplicaAddress>) -> Result<()> {
+ self.repo.heartbeat(replica, address).await
+ }
+
+ async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {
+ self.repo.manager_of(session).await
+ }
+}
+
impl AgentSessionLogRepo for BlockingPromptLogs {
+ async fn create_fenced(
+ &self,
+ log: AgentSessionLog,
+ claim: &SessionClaim,
+ ) -> Result<StoredAgentSessionLog> {
+ self.repo.create_fenced(log, claim).await
+ }
+
async fn create(&self, log: AgentSessionLog) -> Result<StoredAgentSessionLog> {
if self.hang_disconnect
&& matches!(
@@ -512,6 +557,7 @@ async fn close_claiming_an_attach_reservation_prevents_actor_start() {
.await
.expect("attach reserves before reading");
let session = fx.repo.get(fx.session).await.expect("session exists");
+ let claim = claim_for_test(&fx.repo, fx.session).await;
let (stopped, marker) = fx.service.begin_stop(fx.session, false);
let result = fx
@@ -520,6 +566,7 @@ async fn close_claiming_an_attach_reservation_prevents_actor_start() {
session,
RuntimeAttachment::solo(PendingTransport),
reservation,
+ claim,
)
.await;
AgentSessionServiceImpl::<
@@ -552,6 +599,7 @@ async fn shutdown_prevents_a_reserved_attach_from_spawning() {
.await
.expect("attach reserves before reading");
let session = fx.repo.get(fx.session).await.expect("session exists");
+ let claim = claim_for_test(&fx.repo, fx.session).await;
fx.service.shutdown().await;
let result = fx
@@ -560,6 +608,7 @@ async fn shutdown_prevents_a_reserved_attach_from_spawning() {
session,
RuntimeAttachment::solo(PendingTransport),
reservation,
+ claim,
)
.await;
@@ -583,6 +632,30 @@ async fn close_does_not_remove_a_concurrent_delete_guard() {
});
}
+/// The cross-replica half of what `AlreadyConnected` guards in-process: a
+/// second service instance over the same store - two replicas, in production
+/// - cannot attach a session whose managing replica is live. Its claim comes
+/// back `ManagedElsewhere` and the attach refuses before touching the actor.
+#[tokio::test]
+async fn a_second_replica_cannot_attach_a_session_with_a_live_manager() {
+ let fx = fixture();
+ fx.service
+ .attach_session(fx.session, RuntimeAttachment::solo(PendingTransport))
+ .await
+ .expect("first replica attaches");
+
+ let second_replica = AgentSessionServiceImpl::new(
+ fx.repo.clone(),
+ FoldedMessageService::new(fx.repo.clone()),
+ NoOpRealtime,
+ );
+ let result = second_replica
+ .attach_session(fx.session, RuntimeAttachment::solo(PendingTransport))
+ .await;
+
+ assert!(matches!(result, Err(AgentSessionError::ManagedElsewhere(id)) if id == fx.session));
+}
+
/// A command sent while the handshake never completes cannot hang its caller
/// forever - see [`HANDSHAKE_TIMEOUT`]. Without that bound, this test would
/// simply never finish. The actor it was stuck in cannot linger afterwards
@@ -653,12 +726,15 @@ async fn cancellation_does_not_drop_an_effect_batch_after_machine_mutation() {
let cancellation = CancellationToken::new();
let marker = Arc::new(());
let (stopped_tx, _) = watch::channel(false);
+ let claim = claim_for_test(&repo, session).await;
let task = tokio::spawn(run_session(
actor,
Arc::downgrade(&active),
marker,
stopped_tx,
cancellation.clone(),
+ repo.clone(),
+ claim,
));
open_test_session(&inbound_tx, &mut outbound_rx, session).await;
@@ -726,6 +802,7 @@ async fn live_inbound_logs_do_not_reuse_the_expired_handshake_deadline() {
let active = Arc::new(ActiveSessions::new());
let cancellation = CancellationToken::new();
let (stopped_tx, _) = watch::channel(false);
+ let claim = claim_for_test(&repo, session).await;
let task = tokio::spawn(
run_session(
actor,
@@ -733,6 +810,8 @@ async fn live_inbound_logs_do_not_reuse_the_expired_handshake_deadline() {
Arc::new(()),
stopped_tx,
cancellation.clone(),
+ repo.clone(),
+ claim,
)
.with_current_subscriber(),
);
diff --git a/crates/agent_session/src/outbound/postgres.rs b/crates/agent_session/src/outbound/postgres.rs
index d131311651..ab02baf791 100644
--- a/crates/agent_session/src/outbound/postgres.rs
+++ b/crates/agent_session/src/outbound/postgres.rs
@@ -7,10 +7,14 @@ mod test;
use crate::domain::error::{AgentSessionError, Result};
use crate::domain::model::{
- AgentSession, AgentSessionId, AgentSessionLog, ChannelSession, CreateAgentSessionParams,
- ExternalSession, Message, SandboxSize, SessionBot, SessionStatus, StoredAgentSessionLog,
+ AgentSession, AgentSessionId, AgentSessionLog, ChannelSession, ClaimOutcome,
+ CreateAgentSessionParams, ExternalSession, ManagerFence, Message, ReplicaAddress, ReplicaId,
+ SandboxSize, SessionBot, SessionClaim, SessionManager, SessionStatus, StoredAgentSessionLog,
+};
+use crate::domain::ports::{
+ AgentSessionLogRepo, AgentSessionRepo, ExternalSessionRepo, REPLICA_STALE_AFTER,
+ SessionOwnership,
};
-use crate::domain::ports::{AgentSessionLogRepo, AgentSessionRepo, ExternalSessionRepo};
use crate::outbound::connection_gateway_realtime::SessionAudience;
use agent_client_protocol::schema::v1::SessionId;
use agent_runtime_protocol::domain::schema::v0::{SystemEvent, ToRuntimeMessage, ToServerMessage};
@@ -767,6 +771,81 @@ impl AgentSessionLogRepo for PgAgentSessionRepo {
})
}
+ async fn create_fenced(
+ &self,
+ log: AgentSessionLog,
+ claim: &SessionClaim,
+ ) -> Result<StoredAgentSessionLog> {
+ let event_status = match &log.content {
+ Message::ToServer(ToServerMessage::Event { event }) => {
+ Some(SessionStatus::Event(event.clone()))
+ }
+ _ => None,
+ };
+ let (direction, content) = message_columns(&log.content)?;
+ let mut transaction = self
+ .pool
+ .begin()
+ .await
+ .context("begin fenced agent session log create")?;
+ // The fencing contract: the guard and the append are one statement,
+ // so there is no instant between "checked we still hold the lease"
+ // and "wrote" for a takeover to slip into.
+ let created_at = sqlx::query_scalar!(
+ r#"
+ INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content)
+ SELECT $1, $2, $3, $4, $5
+ WHERE EXISTS (
+ SELECT 1 FROM agent_session
+ WHERE id = $2 AND manager_replica_id = $6 AND manager_fence = $7
+ )
+ RETURNING created_at
+ "#,
+ macro_uuid::generate_uuid_v7(),
+ log.agent_session_id.as_uuid(),
+ log.user_id.as_ref().map(|user_id| user_id.as_ref()),
+ direction,
+ content,
+ claim.replica.as_uuid(),
+ claim.fence.0,
+ )
+ .fetch_optional(&mut *transaction)
+ .await
+ .context("failed to create fenced agent session log entry")?;
+ let Some(created_at) = created_at else {
+ return Err(AgentSessionError::FencedOut(log.agent_session_id));
+ };
+
+ if let Some(status) = event_status {
+ let (status, status_event_name) = status_columns(&status);
+ sqlx::query!(
+ r#"
+ UPDATE agent_session
+ SET status = $2,
+ status_event_name = $3,
+ modified_at = now()
+ WHERE id = $1
+ "#,
+ log.agent_session_id.as_uuid(),
+ status,
+ status_event_name,
+ )
+ .execute(&mut *transaction)
+ .await
+ .context("failed to update agent session status from fenced log entry")?;
+ }
+
+ transaction
+ .commit()
+ .await
+ .context("commit fenced agent session log create")?;
+
+ Ok(StoredAgentSessionLog {
+ created_at,
+ entry: log,
+ })
+ }
+
async fn list_by_session(
&self,
agent_session_id: AgentSessionId,
@@ -797,6 +876,144 @@ impl AgentSessionLogRepo for PgAgentSessionRepo {
}
}
+impl SessionOwnership for PgAgentSessionRepo {
+ async fn claim(&self, session: AgentSessionId, replica: ReplicaId) -> Result<ClaimOutcome> {
+ // One statement: the replica's heartbeat row is upserted in the CTE
+ // (a claim can never reference a replica the store has not seen),
+ // then the lease itself is a compare-and-swap - taken only from
+ // nobody, ourselves, or a stale holder, and every take bumps the
+ // fence so a superseded holder's fenced writes stop matching.
+ let fence = sqlx::query_scalar!(
+ r#"
+ WITH replica AS (
+ INSERT INTO harness_replica (id, last_heartbeat_at)
+ VALUES ($2, now())
+ ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()
+ )
+ UPDATE agent_session
+ SET manager_replica_id = $2,
+ manager_fence = manager_fence + 1,
+ modified_at = now()
+ WHERE id = $1
+ AND (
+ manager_replica_id IS NULL
+ OR manager_replica_id = $2
+ OR NOT EXISTS (
+ SELECT 1 FROM harness_replica live
+ WHERE live.id = agent_session.manager_replica_id
+ AND live.last_heartbeat_at > now() - make_interval(secs => $3)
+ )
+ )
+ RETURNING manager_fence
+ "#,
+ session.as_uuid(),
+ replica.as_uuid(),
+ REPLICA_STALE_AFTER.as_secs_f64(),
+ )
+ .fetch_optional(&self.pool)
+ .await
+ .context("failed to claim agent session management")?;
+
+ if let Some(fence) = fence {
+ return Ok(ClaimOutcome::Claimed(SessionClaim {
+ session,
+ replica,
+ fence: ManagerFence(fence),
+ }));
+ }
+
+ // The swap matched nothing: either a live replica holds the lease, or
+ // the session row is gone. A deleted session reads as Unknown so the
+ // caller does not forward commands toward a row that no longer exists.
+ let holder = sqlx::query_scalar!(
+ r#"SELECT manager_replica_id FROM agent_session WHERE id = $1"#,
+ session.as_uuid(),
+ )
+ .fetch_optional(&self.pool)
+ .await
+ .context("failed to read the agent session manager")?;
+ match holder {
+ Some(Some(holder)) => Ok(ClaimOutcome::ManagedElsewhere(ReplicaId::from_uuid(holder))),
+ Some(None) | None => Err(AgentSessionError::Unknown(anyhow::anyhow!(
+ "agent session {session} claim matched nothing yet no live replica holds it"
+ ))),
+ }
+ }
+
+ async fn release(&self, claim: &SessionClaim) -> Result<()> {
+ // Conditional on both holder and fence: a release arriving after a
+ // successor claimed must not free the successor's lease, and a
+ // deleted session matches nothing, which is the asked-for state.
+ sqlx::query!(
+ r#"
+ UPDATE agent_session
+ SET manager_replica_id = NULL,
+ modified_at = now()
+ WHERE id = $1 AND manager_replica_id = $2 AND manager_fence = $3
+ "#,
+ claim.session.as_uuid(),
+ claim.replica.as_uuid(),
+ claim.fence.0,
+ )
+ .execute(&self.pool)
+ .await
+ .context("failed to release agent session management")?;
+ Ok(())
+ }
+
+ async fn heartbeat(&self, replica: ReplicaId, address: Option<&ReplicaAddress>) -> Result<()> {
+ // COALESCE keeps a previously published address when a beat carries
+ // none, so a claim-created row filled in by one heartbeat is not
+ // blanked by the next.
+ sqlx::query!(
+ r#"
+ INSERT INTO harness_replica (id, last_heartbeat_at, address)
+ VALUES ($1, now(), $2)
+ ON CONFLICT (id) DO UPDATE
+ SET last_heartbeat_at = now(),
+ address = COALESCE(EXCLUDED.address, harness_replica.address)
+ "#,
+ replica.as_uuid(),
+ address.map(ReplicaAddress::as_str),
+ )
+ .execute(&self.pool)
+ .await
+ .context("failed to heartbeat harness replica")?;
+ // Housekeeping on the writer that is already here: rows a week past
+ // their last heartbeat are boots nothing can still reference usefully
+ // (their claims were stealable within seconds); the FK sets any
+ // stragglers' claims to NULL.
+ sqlx::query!(
+ r#"DELETE FROM harness_replica WHERE last_heartbeat_at < now() - interval '7 days'"#,
+ )
+ .execute(&self.pool)
+ .await
+ .context("failed to prune stale harness replicas")?;
+ Ok(())
+ }
+
+ async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {
+ let row = sqlx::query!(
+ r#"
+ SELECT live.id AS replica_id, live.address
+ FROM agent_session
+ JOIN harness_replica live ON live.id = agent_session.manager_replica_id
+ WHERE agent_session.id = $1
+ AND live.last_heartbeat_at > now() - make_interval(secs => $2)
+ "#,
+ session.as_uuid(),
+ REPLICA_STALE_AFTER.as_secs_f64(),
+ )
+ .fetch_optional(&self.pool)
+ .await
+ .context("failed to read the agent session's live manager")?;
+ Ok(row.map(|row| SessionManager {
+ replica: ReplicaId::from_uuid(row.replica_id),
+ address: row.address.map(ReplicaAddress::new),
+ }))
+ }
+}
+
/// Folding reads the log through `agent_fold`'s own port; this adapter
/// already speaks [`AgentSessionLogRepo`], so bridging is one line.
impl agent_fold::domain::ports::LogRepo for PgAgentSessionRepo {
diff --git a/crates/agent_session/src/outbound/postgres/test.rs b/crates/agent_session/src/outbound/postgres/test.rs
index 61e917cdc0..1d20c178c7 100644
--- a/crates/agent_session/src/outbound/postgres/test.rs
+++ b/crates/agent_session/src/outbound/postgres/test.rs
@@ -892,3 +892,138 @@ async fn deleting_a_session_cascades_its_external_row(pool: PgPool) {
None
);
}
+
+/// Backdate a replica's heartbeat far past `REPLICA_STALE_AFTER`, so its
+/// claims read as up for grabs.
+async fn let_heartbeat_go_stale(pool: &PgPool, replica: ReplicaId) {
+ sqlx::query!(
+ r#"UPDATE harness_replica SET last_heartbeat_at = now() - interval '10 minutes' WHERE id = $1"#,
+ replica.as_uuid(),
+ )
+ .execute(pool)
+ .await
+ .expect("backdate replica heartbeat");
+}
+
+fn claimed(outcome: ClaimOutcome) -> SessionClaim {
+ match outcome {
+ ClaimOutcome::Claimed(claim) => claim,
+ ClaimOutcome::ManagedElsewhere(holder) => {
+ panic!("expected to claim, but {holder} manages the session")
+ }
+ }
+}
+
+fn fenced_log(id: AgentSessionId) -> AgentSessionLog {
+ AgentSessionLog {
+ agent_session_id: id,
+ user_id: None,
+ content: Message::ToRuntime(ToRuntimeMessage::Acp(acp_notification())),
+ }
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn claiming_is_reentrant_and_every_claim_bumps_the_fence(pool: PgPool) {
+ let repo = PgAgentSessionRepo::new(pool.clone());
+ let bot_id = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot_id, None, None)).await;
+ let replica = ReplicaId::mint();
+
+ let first = claimed(repo.claim(session.id, replica).await.expect("first claim"));
+ let second = claimed(repo.claim(session.id, replica).await.expect("second claim"));
+
+ assert_eq!(first.fence, ManagerFence(1));
+ assert_eq!(second.fence, ManagerFence(2));
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_live_holder_blocks_a_second_replica(pool: PgPool) {
+ let repo = PgAgentSessionRepo::new(pool.clone());
+ let bot_id = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot_id, None, None)).await;
+ let holder = ReplicaId::mint();
+ let contender = ReplicaId::mint();
+
+ claimed(repo.claim(session.id, holder).await.expect("claim"));
+ match repo.claim(session.id, contender).await.expect("contend") {
+ ClaimOutcome::ManagedElsewhere(seen) => assert_eq!(seen, holder),
+ ClaimOutcome::Claimed(_) => panic!("a live holder's claim was stolen"),
+ }
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_stale_holder_is_superseded(pool: PgPool) {
+ let repo = PgAgentSessionRepo::new(pool.clone());
+ let bot_id = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot_id, None, None)).await;
+ let crashed = ReplicaId::mint();
+ let successor = ReplicaId::mint();
+
+ claimed(repo.claim(session.id, crashed).await.expect("claim"));
+ let_heartbeat_go_stale(&pool, crashed).await;
+
+ let takeover = claimed(repo.claim(session.id, successor).await.expect("takeover"));
+ assert_eq!(takeover.fence, ManagerFence(2));
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn release_frees_the_lease_but_never_a_successors(pool: PgPool) {
+ let repo = PgAgentSessionRepo::new(pool.clone());
+ let bot_id = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot_id, None, None)).await;
+ let crashed = ReplicaId::mint();
+ let successor = ReplicaId::mint();
+ let third = ReplicaId::mint();
+
+ let superseded = claimed(repo.claim(session.id, crashed).await.expect("claim"));
+ let_heartbeat_go_stale(&pool, crashed).await;
+ let current = claimed(repo.claim(session.id, successor).await.expect("takeover"));
+
+ // The superseded holder's release is a no-op: the successor still holds.
+ repo.release(&superseded).await.expect("stale release");
+ match repo.claim(session.id, third).await.expect("contend") {
+ ClaimOutcome::ManagedElsewhere(seen) => assert_eq!(seen, successor),
+ ClaimOutcome::Claimed(_) => panic!("a stale release freed the successor's lease"),
+ }
+
+ // The current holder's release frees it for anyone.
+ repo.release(&current).await.expect("current release");
+ claimed(repo.claim(session.id, third).await.expect("reclaim"));
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_fenced_append_rejects_a_superseded_writer(pool: PgPool) {
+ let repo = PgAgentSessionRepo::new(pool.clone());
+ let bot_id = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot_id, None, None)).await;
+ let zombie = ReplicaId::mint();
+ let successor = ReplicaId::mint();
+
+ let old_claim = claimed(repo.claim(session.id, zombie).await.expect("claim"));
+ repo.create_fenced(fenced_log(session.id), &old_claim)
+ .await
+ .expect("the live holder's fenced append lands");
+
+ let_heartbeat_go_stale(&pool, zombie).await;
+ let new_claim = claimed(repo.claim(session.id, successor).await.expect("takeover"));
+
+ // The zombie wakes and writes under its superseded fence: rejected by the
+ // same statement that would have written, no matter how it got confused.
+ match repo.create_fenced(fenced_log(session.id), &old_claim).await {
+ Err(AgentSessionError::FencedOut(id)) => assert_eq!(id, session.id),
+ other => panic!("a superseded fence wrote anyway: {other:?}"),
+ }
+
+ repo.create_fenced(fenced_log(session.id), &new_claim)
+ .await
+ .expect("the successor's fenced append lands");
+
+ let log = AgentSessionLogRepo::list_by_session(&repo, session.id)
+ .await
+ .expect("list log");
+ assert_eq!(
+ log.len(),
+ 2,
+ "exactly the two live-holder appends are in the log"
+ );
+}
diff --git a/crates/agent_session/src/testing.rs b/crates/agent_session/src/testing.rs
index 622f2aaf1b..578476df99 100644
--- a/crates/agent_session/src/testing.rs
+++ b/crates/agent_session/src/testing.rs
@@ -6,11 +6,15 @@
use crate::domain::error::{AgentSessionError, Result};
use crate::domain::model::{
- AgentSession, AgentSessionId, AgentSessionLog, ChannelSession, CreateAgentSessionParams,
- DEFAULT_AGENT_SESSION_NAME, LogAppended, SandboxSize, SessionBot, SessionStatus,
- StoredAgentSessionLog,
+ AgentSession, AgentSessionId, AgentSessionLog, ChannelSession, ClaimOutcome,
+ CreateAgentSessionParams, DEFAULT_AGENT_SESSION_NAME, LogAppended, ManagerFence,
+ ReplicaAddress, ReplicaId, SandboxSize, SessionBot, SessionClaim, SessionManager,
+ SessionStatus, StoredAgentSessionLog,
+};
+use crate::domain::ports::{
+ AgentSessionLogRepo, AgentSessionRealtime, AgentSessionRepo, REPLICA_STALE_AFTER,
+ SessionOwnership,
};
-use crate::domain::ports::{AgentSessionLogRepo, AgentSessionRealtime, AgentSessionRepo};
use agent_client_protocol::schema::v1::SessionId;
use agent_runtime_protocol::domain::schema::v0::ToServerMessage;
use bots::domain::models::BotId;
@@ -20,6 +24,13 @@ use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
+/// One session's lease state: the holding replica (if any) and the fence,
+/// which outlives the holder as in the real schema.
+type Lease = (Option<ReplicaId>, i64);
+
+/// One replica's row: its last heartbeat and published forwarding address.
+type ReplicaRow = (std::time::Instant, Option<ReplicaAddress>);
+
/// An in-memory [`AgentSessionRepo`] and [`AgentSessionLogRepo`].
///
/// Cheap to clone - clones share one store, so a handle kept for assertions
@@ -36,6 +47,11 @@ pub struct InMemoryAgentSessionRepo {
user_sizes: Arc<Mutex<HashMap<String, SandboxSize>>>,
log_reads: Arc<AtomicUsize>,
session_reads: Arc<AtomicUsize>,
+ /// Replica heartbeats and published addresses, mirroring `harness_replica`.
+ replicas: Arc<Mutex<HashMap<ReplicaId, ReplicaRow>>>,
+ /// Session -> lease, mirroring the lease columns: release clears the
+ /// holder and leaves the counter.
+ leases: Arc<Mutex<HashMap<AgentSessionId, Lease>>>,
}
impl InMemoryAgentSessionRepo {
@@ -318,6 +334,99 @@ impl AgentSessionRepo for InMemoryAgentSessionRepo {
}
}
+impl SessionOwnership for InMemoryAgentSessionRepo {
+ async fn claim(&self, session: AgentSessionId, replica: ReplicaId) -> Result<ClaimOutcome> {
+ if !self
+ .sessions
+ .lock()
+ .expect("in-memory session store is not poisoned")
+ .contains_key(&session)
+ {
+ return Err(AgentSessionError::Unknown(anyhow::anyhow!(
+ "agent session {session} does not exist to claim"
+ )));
+ }
+ let now = std::time::Instant::now();
+ let mut replicas = self
+ .replicas
+ .lock()
+ .expect("in-memory replica store is not poisoned");
+ replicas.entry(replica).or_insert((now, None)).0 = now;
+ let mut leases = self
+ .leases
+ .lock()
+ .expect("in-memory lease store is not poisoned");
+ let (holder, fence) = leases.entry(session).or_insert((None, 0));
+ let holder_is_live = holder.filter(|holder| *holder != replica).filter(|holder| {
+ replicas
+ .get(holder)
+ .is_some_and(|(beat, _)| now.duration_since(*beat) < REPLICA_STALE_AFTER)
+ });
+ if let Some(holder) = holder_is_live {
+ return Ok(ClaimOutcome::ManagedElsewhere(holder));
+ }
+ *holder = Some(replica);
+ *fence += 1;
+ Ok(ClaimOutcome::Claimed(SessionClaim {
+ session,
+ replica,
+ fence: ManagerFence(*fence),
+ }))
+ }
+
+ async fn release(&self, claim: &SessionClaim) -> Result<()> {
+ let mut leases = self
+ .leases
+ .lock()
+ .expect("in-memory lease store is not poisoned");
+ if let Some((holder, fence)) = leases.get_mut(&claim.session)
+ && *holder == Some(claim.replica)
+ && *fence == claim.fence.0
+ {
+ *holder = None;
+ }
+ Ok(())
+ }
+
+ async fn heartbeat(&self, replica: ReplicaId, address: Option<&ReplicaAddress>) -> Result<()> {
+ let mut replicas = self
+ .replicas
+ .lock()
+ .expect("in-memory replica store is not poisoned");
+ let entry = replicas
+ .entry(replica)
+ .or_insert((std::time::Instant::now(), None));
+ entry.0 = std::time::Instant::now();
+ // As in the real adapter: a beat carrying no address keeps the one
+ // already published.
+ if let Some(address) = address {
+ entry.1 = Some(address.clone());
+ }
+ Ok(())
+ }
+
+ async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {
+ let leases = self
+ .leases
+ .lock()
+ .expect("in-memory lease store is not poisoned");
+ let Some((Some(holder), _)) = leases.get(&session) else {
+ return Ok(None);
+ };
+ let replicas = self
+ .replicas
+ .lock()
+ .expect("in-memory replica store is not poisoned");
+ Ok(replicas
+ .get(holder)
+ .filter(|(beat, _)| beat.elapsed() < REPLICA_STALE_AFTER)
+ .map(|(_, address)| SessionManager {
+ replica: *holder,
+ address: address.clone(),
+ }))
+ }
+}
+
impl AgentSessionLogRepo for InMemoryAgentSessionRepo {
async fn create(&self, log: AgentSessionLog) -> Result<StoredAgentSessionLog> {
let model_change = match &log.content {
@@ -367,6 +476,28 @@ impl AgentSessionLogRepo for InMemoryAgentSessionRepo {
Ok(stored)
}
+ async fn create_fenced(
+ &self,
+ log: AgentSessionLog,
+ claim: &SessionClaim,
+ ) -> Result<StoredAgentSessionLog> {
+ let fenced_out = {
+ let leases = self
+ .leases
+ .lock()
+ .expect("in-memory lease store is not poisoned");
+ !matches!(
+ leases.get(&log.agent_session_id),
+ Some((holder, fence))
+ if *holder == Some(claim.replica) && *fence == claim.fence.0
+ )
+ };
+ if fenced_out {
+ return Err(AgentSessionError::FencedOut(log.agent_session_id));
+ }
+ AgentSessionLogRepo::create(self, log).await
+ }
+
async fn list_by_session(
&self,
agent_session_id: AgentSessionId,
diff --git a/crates/macro_db_client/migrations/20260828161717_agent_session_manager_lease.sql b/crates/macro_db_client/migrations/20260828161717_agent_session_manager_lease.sql
new file mode 100644
index 0000000000..f46553fe8b
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260828161717_agent_session_manager_lease.sql
@@ -0,0 +1,22 @@
+-- Which harness replica holds a session's live actor, as a lease.
+--
+-- One row per booted harness participant, heartbeated while it lives. A
+-- replica that stops heartbeating is dead by definition: its claims become
+-- claimable by anyone, so a crashed process releases everything at once
+-- without per-session cleanup.
+CREATE TABLE harness_replica (
+ id UUID PRIMARY KEY,
+ started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ last_heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- The lease itself lives on the session so claiming and deleting are
+-- transactional with the row they protect. manager_fence is a takeover
+-- counter: every successful claim increments it, and live-actor log appends
+-- are conditioned on the fence they were attached under, so a replica that
+-- stalls past its heartbeat and gets superseded has its writes rejected by
+-- the store rather than interleaved (fencing tokens). The fence never
+-- resets - ON DELETE SET NULL frees the claim but keeps the counter.
+ALTER TABLE agent_session
+ ADD COLUMN manager_replica_id UUID REFERENCES harness_replica(id) ON DELETE SET NULL,
+ ADD COLUMN manager_fence BIGINT NOT NULL DEFAULT 0;
diff --git a/crates/macro_db_client/migrations/20260828183712_harness_replica_address.sql b/crates/macro_db_client/migrations/20260828183712_harness_replica_address.sql
new file mode 100644
index 0000000000..f63058ae4b
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260828183712_harness_replica_address.sql
@@ -0,0 +1,8 @@
+-- Where to forward a session's commands when another replica manages it:
+-- the replica's own private base URL, written with its heartbeat. Nullable
+-- because a replica row can be created by a claim before the first
+-- heartbeat carries the address, and because a local single-replica stack
+-- has no routable address to publish - a peer that reads NULL treats the
+-- manager as unreachable rather than guessing.
+ALTER TABLE harness_replica
+ ADD COLUMN address TEXT;
diff --git a/infra/stacks/agent-harness-service/agent_harness_service.ts b/infra/stacks/agent-harness-service/agent_harness_service.ts
index 794583e615..8e89628c2e 100644
--- a/infra/stacks/agent-harness-service/agent_harness_service.ts
+++ b/infra/stacks/agent-harness-service/agent_harness_service.ts
@@ -59,12 +59,25 @@ type Args = {
};
/**
- * The agent harness service. Like agent-proxy before it, this component pins
- * `desiredCount` to 1 and forces a stop-then-start deployment (min healthy
- * 0%, max 100%) with no autoscaling: the harness owns the live agent-session
- * actors in process memory with no cross-instance sync, and its Kafka consumer
- * groups must not split partitions across two momentarily-coexisting tasks
- * (see the consumer groups in `services/agent_harness_service`).
+ * The agent harness service. Replicated in every environment: each replica
+ * claims the sessions whose live actors it holds through a fenced Postgres
+ * lease, and a command landing on the wrong replica is forwarded task-to-task
+ * over the private network (`POST /internal/agent-sessions/{id}/command`,
+ * internal-key authenticated) - which is why the service security group
+ * allows itself ingress on the service port. The Kafka consumer group splits
+ * partitions across live tasks; the lease plus forwarding is what makes that
+ * split correct.
+ *
+ * Deploys roll (min healthy 100%, max 200%): the outgoing tasks keep serving
+ * until their replacements pass health checks, so the control API and egress
+ * proxy stay up instead of blacking out for the old tasks' sandbox cleanup.
+ * A deploy therefore doubles the replica count for its duration rather than
+ * introducing coexistence - replicas coexist all the time, and the lease is
+ * what makes both the steady state and the overlap correct. A command routed
+ * to a new task for a session whose live actor is still on an old one
+ * self-heals through resume. Live sessions already restart across every
+ * deploy (`shutdown_all` stops each sandbox), so the overlap narrows the
+ * blast radius rather than widening it.
*/
export class AgentHarnessService extends pulumi.ComponentResource {
public role: aws.iam.Role;
@@ -310,18 +323,20 @@ export class AgentHarnessService extends pulumi.ComponentResource {
enable: true,
rollback: true,
},
- // Never run 2 tasks at once, even transiently during a deploy: stop
- // the old one before the new one starts (see the class doc comment).
- // This blackout covers the egress proxy too - sandbox git and MCP
- // calls fail for the whole window, they are not more available than
- // the control API.
- deploymentMinimumHealthyPercent: 0,
- deploymentMaximumPercent: 100,
- desiredCount: 1,
+ // Rolling replace: the old task serves until the new one is healthy
+ // (see the class doc comment for the overlap semantics). A failed
+ // deploy rolls back with the old task still up rather than at zero.
+ deploymentMinimumHealthyPercent: 100,
+ deploymentMaximumPercent: 200,
+ // Every environment runs two, so dev stays prod-shaped: forwarding is
+ // dead code at one task (management() can only answer Ours or
+ // Unmanaged), and a path only dev never exercises is one whose
+ // regressions surface in prod.
+ desiredCount: 2,
// ALB checks /health every 10s and fails the target after two
// misses (~20s). HTTP does not listen until after DB, AWS config,
// and JWT secrets, so a 0s grace period trips the circuit breaker
- // on this stop-then-start replace. Ignore those checks until bind.
+ // before the replacement binds. Ignore those checks until then.
healthCheckGracePeriodSeconds: 120,
taskDefinitionArgs: {
taskRole: {
@@ -473,6 +488,24 @@ export class AgentHarnessService extends pulumi.ComponentResource {
{ parent: this }
);
+ // Replica-to-replica command forwarding: a task that consumed a command
+ // for a session another replica manages POSTs it directly to that
+ // replica's private address, bypassing the load balancer.
+ new aws.vpc.SecurityGroupIngressRule(
+ `${BASE_NAME}-peer-in`,
+ {
+ securityGroupId: serviceSg.id,
+ description:
+ 'Allow harness replicas to forward session commands to each other',
+ referencedSecurityGroupId: serviceSg.id,
+ fromPort: serviceContainerPort,
+ toPort: serviceContainerPort,
+ ipProtocol: 'tcp',
+ tags: this.tags,
+ },
+ { parent: this }
+ );
+
new aws.vpc.SecurityGroupEgressRule(
`${BASE_NAME}-service-out`,
{
diff --git a/services/agent_harness_service/Cargo.toml b/services/agent_harness_service/Cargo.toml
index 0864294d16..6baa4e0601 100644
--- a/services/agent_harness_service/Cargo.toml
+++ b/services/agent_harness_service/Cargo.toml
@@ -61,6 +61,7 @@ macro_authorization = { path = "../../crates/macro_authorization", default-featu
] }
macro_cors = { path = "../../crates/macro_cors" }
secretsmanager_client = { path = "../../crates/secretsmanager_client" }
+reqwest = { workspace = true }
serde_json = { workspace = true }
utoipa = { workspace = true }
utoipa-swagger-ui = { workspace = true }
diff --git a/services/agent_harness_service/src/api.rs b/services/agent_harness_service/src/api.rs
index b9739d5506..ddf3807c2f 100644
--- a/services/agent_harness_service/src/api.rs
+++ b/services/agent_harness_service/src/api.rs
@@ -7,6 +7,8 @@ use std::time::Duration;
use agent_egress::domain::service::EgressService;
use agent_egress::inbound::axum_router::{EgressRouterState, egress_router};
+use agent_harness::domain::service::ForwardedCommands;
+use agent_harness::inbound::forward::{ForwardGatewayState, forward_router};
use agent_harness::inbound::runtime_gateway::{RuntimeGatewayState, runtime_gateway_router};
use agent_session::domain::ports::{
AgentSessionNotificationRecipient, BotDirectory, SessionOpener,
@@ -55,11 +57,12 @@ where
}
/// Build the router and serve it until the process is asked to stop.
-pub async fn setup_and_serve<T, R, Opener, Bots, Access, Auth>(
+pub async fn setup_and_serve<T, R, Opener, Bots, Access, Auth, Harness>(
read_state: AgentSessionRouterState<T, Access, Auth>,
control_state: AgentSessionControlState<R, Access, Auth>,
create_state: CreateSessionState<Opener, Bots, Auth>,
gateway_state: RuntimeGatewayState<Bots, Auth>,
+ forward_state: ForwardGatewayState<Harness, Auth>,
port: u16,
shutdown: impl Future<Output = ()> + Send + 'static,
) -> anyhow::Result<()>
@@ -70,12 +73,19 @@ where
Bots: BotDirectory,
Access: EntityAccessService,
Auth: MacroAuthorizationService,
+ Harness: ForwardedCommands,
{
- let app = api_router(read_state, control_state, create_state, gateway_state)
- .layer(MacroRequestIdAndTracingLayer::new(Duration::from_millis(200)).into_inner())
- .merge(Router::new().route("/health", get(health)))
- .layer(macro_cors::cors_layer())
- .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()));
+ let app = api_router(
+ read_state,
+ control_state,
+ create_state,
+ gateway_state,
+ forward_state,
+ )
+ .layer(MacroRequestIdAndTracingLayer::new(Duration::from_millis(200)).into_inner())
+ .merge(Router::new().route("/health", get(health)))
+ .layer(macro_cors::cors_layer())
+ .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", swagger::ApiDoc::openapi()));
let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}"))
.await
@@ -89,11 +99,12 @@ where
.context("agent harness service http failed")
}
-fn api_router<T, R, Opener, Bots, Access, Auth>(
+fn api_router<T, R, Opener, Bots, Access, Auth, Harness>(
read_state: AgentSessionRouterState<T, Access, Auth>,
control_state: AgentSessionControlState<R, Access, Auth>,
create_state: CreateSessionState<Opener, Bots, Auth>,
gateway_state: RuntimeGatewayState<Bots, Auth>,
+ forward_state: ForwardGatewayState<Harness, Auth>,
) -> Router
where
T: AgentSessionService,
@@ -102,6 +113,7 @@ where
Bots: BotDirectory,
Access: EntityAccessService,
Auth: MacroAuthorizationService,
+ Harness: ForwardedCommands,
{
let agent_sessions = agent_session_read_router(read_state.clone())
.merge(agent_session_control_router(control_state))
@@ -110,6 +122,11 @@ where
.nest("/agent-sessions", agent_sessions)
.merge(agent_sandbox_size_router(read_state))
.nest("/runtime", runtime_gateway_router(gateway_state))
+ // Replica-to-replica command forwarding. Internal-key authenticated,
+ // and reached over task-to-task networking rather than the load
+ // balancer; mounting it on the public listener is fine because the
+ // extractor refuses anything without the deployment's internal key.
+ .nest("/internal", forward_router(forward_state))
}
async fn health() -> &'static str {
diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rs
index e64412d357..9da1972d60 100644
--- a/services/agent_harness_service/src/main.rs
+++ b/services/agent_harness_service/src/main.rs
@@ -24,6 +24,7 @@ use agent_egress::outbound::session_authority::StoredTokenSessionAuthority;
use agent_fold::domain::service::FoldedMessageService;
use agent_harness::domain::model::{HarnessCommand, HarnessDefaults, SessionDefaults};
use agent_harness::domain::service::AgentHarnessService;
+use agent_harness::inbound::forward::ForwardGatewayState;
use agent_harness::inbound::kafka::{RoutedTrigger, route_agent_trigger};
use agent_harness::inbound::runtime_gateway::RuntimeGatewayState;
use agent_harness::outbound::agent_prompt_composer::LexicalAgentPromptComposer;
@@ -36,13 +37,15 @@ use agent_harness::outbound::daytona::{
DaytonaContainerManager, DaytonaSettings, Snapshot,
};
use agent_harness::outbound::egress::EgressProvisioner;
+use agent_harness::outbound::forward::HttpCommandForwarder;
use agent_harness::outbound::local::{LocalContainerManager, LocalSettings};
use agent_harness::outbound::routing::RoutedContainerManager;
use agent_harness::outbound::runtime_registry::RuntimeRegistry;
use agent_inmem::outbound::log_frames::LogFrameSource;
use agent_inmem::outbound::manager::InMemAgentManager;
use agent_inmem::outbound::rig_engine::RigTurnEngine;
-use agent_session::domain::ports::NoOpRealtime;
+use agent_session::domain::model::{ReplicaAddress, ReplicaId};
+use agent_session::domain::ports::{NoOpRealtime, SessionOwnership as _};
use agent_session::domain::service::AgentSessionServiceImpl;
use agent_session::inbound::axum_router::{
AgentSessionControlState, AgentSessionRouterState, CreateSessionState,
@@ -125,6 +128,63 @@ struct PendingHarnessWork {
description: &'static str,
}
+/// This task's own private base URL, for peers forwarding session commands.
+///
+/// On ECS the task metadata endpoint (v4) publishes the task's private IPv4;
+/// the port is the service listener peers can reach through the task-to-task
+/// security group rule. Local development has no routable address and runs
+/// one replica, so `None` is correct there; every other environment refuses
+/// to boot rather than heartbeat an unreachable identity, because a replica
+/// without an address is one whose sessions peers can see but never reach.
+async fn discover_replica_address(
+ environment: Environment,
+ port: u16,
+) -> anyhow::Result<Option<agent_session::domain::model::ReplicaAddress>> {
+ // Explicit override first: how a local multi-replica experiment names
+ // each process (`REPLICA_ADDRESS=http://127.0.0.1:<port>`), since nothing
+ // off ECS can discover an address.
+ if let Some(address) = macro_env_var::optional_read_env_var("REPLICA_ADDRESS")
+ .ok()
+ .flatten()
+ {
+ return Ok(Some(agent_session::domain::model::ReplicaAddress::new(
+ address,
+ )));
+ }
+ let Some(metadata_uri) = macro_env_var::optional_read_env_var("ECS_CONTAINER_METADATA_URI_V4")
+ .ok()
+ .flatten()
+ else {
+ if matches!(environment, Environment::Local) {
+ return Ok(None);
+ }
+ anyhow::bail!(
+ "ECS_CONTAINER_METADATA_URI_V4 is unset outside a local environment: \
+ this replica would be unreachable for command forwarding"
+ );
+ };
+ let task: serde_json::Value = reqwest::Client::new()
+ .get(format!("{metadata_uri}/task"))
+ .timeout(std::time::Duration::from_secs(5))
+ .send()
+ .await
+ .context("failed to reach the ECS task metadata endpoint")?
+ .json()
+ .await
+ .context("failed to parse ECS task metadata")?;
+ let ip = task["Containers"]
+ .as_array()
+ .into_iter()
+ .flatten()
+ .flat_map(|container| container["Networks"].as_array().into_iter().flatten())
+ .flat_map(|network| network["IPv4Addresses"].as_array().into_iter().flatten())
+ .find_map(serde_json::Value::as_str)
+ .context("ECS task metadata carries no private IPv4 address")?;
+ Ok(Some(agent_session::domain::model::ReplicaAddress::new(
+ format!("http://{ip}:{port}"),
+ )))
+}
+
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let entrypoint = MacroEntrypoint::default().init();
@@ -177,6 +237,18 @@ async fn run() -> anyhow::Result<()> {
// log and pushes each frame at the channel's participants so a viewer sees
// it happen.
let session_repo = PgAgentSessionRepo::new(pool.clone());
+ // One lease identity for the whole process: commands forward to an
+ // address, and both attach-capable service instances below live at this
+ // one, so they share the id. The address is this task's own private URL,
+ // discovered from ECS metadata; a deployment that cannot learn it must
+ // not scale, so discovery failing anywhere but local refuses to boot.
+ let replica = ReplicaId::mint();
+ let replica_address = discover_replica_address(config.environment, config.port).await?;
+ tracing::info!(
+ %replica,
+ address = ?replica_address.as_ref().map(ReplicaAddress::as_str),
+ "harness replica identity"
+ );
let sessions = AgentSessionServiceImpl::new(
session_repo.clone(),
FoldedMessageService::new(session_repo.clone()),
@@ -185,6 +257,7 @@ async fn run() -> anyhow::Result<()> {
session_repo.clone(),
),
)
+ .with_replica(replica)
.with_name_generator(HaikuAgentSessionNameGenerator::new(ai_usage::pg_recorder(
pool.clone(),
)));
@@ -260,15 +333,13 @@ async fn run() -> anyhow::Result<()> {
};
// The sandbox provider serves every bot but the in-memory one, which the
// router pulls out by bot id before the provider ever sees it.
- let sandbox_and_inmem = RoutedContainers::new(
- sandbox,
- inmem,
- AgentSessionServiceImpl::new(
- session_repo.clone(),
- FoldedMessageService::new(session_repo.clone()),
- NoOpRealtime,
- ),
- );
+ let inmem_sessions = AgentSessionServiceImpl::new(
+ session_repo.clone(),
+ FoldedMessageService::new(session_repo.clone()),
+ NoOpRealtime,
+ )
+ .with_replica(replica);
+ let sandbox_and_inmem = RoutedContainers::new(sandbox, inmem, inmem_sessions);
// Cursor sessions run on their owner's own Cursor account, so there is no
// deployment-wide key to arm this with: the manager reads each session
@@ -406,6 +477,8 @@ async fn run() -> anyhow::Result<()> {
prompt_context,
prompt_composer,
EgressProvisioner::new(Arc::clone(&mcp_connections), config.egress_base_url.clone()),
+ HttpCommandForwarder::new(config.internal_api_key.clone())
+ .map_err(|error| anyhow::anyhow!("failed to build the command forwarder: {error}"))?,
defaults,
));
@@ -446,6 +519,10 @@ async fn run() -> anyhow::Result<()> {
let gateway_state = RuntimeGatewayState::new(
runtimes,
bots_directory,
+ MacroAuthorizationState::new(Arc::new(authorization_service.clone())),
+ );
+ let forward_state = ForwardGatewayState::new(
+ harness.clone(),
MacroAuthorizationState::new(Arc::new(authorization_service)),
);
let http_port = config.port;
@@ -455,6 +532,7 @@ async fn run() -> anyhow::Result<()> {
control_state,
create_state,
gateway_state,
+ forward_state,
http_port,
shutdown_signal(),
)
@@ -464,6 +542,27 @@ async fn run() -> anyhow::Result<()> {
}
});
+ // The session lease's liveness signal: while this beats, this process's
+ // claims are held; when it stops - crash or shutdown - they go stale
+ // within REPLICA_STALE_AFTER and any successor can claim. Graceful stops
+ // release each claim eagerly in the actor teardown path; this loop is
+ // what covers the ungraceful ones.
+ let heartbeat_repo = session_repo.clone();
+ let heartbeat_address = replica_address.clone();
+ let heartbeat = tokio::spawn(async move {
+ let mut ticker =
+ tokio::time::interval(agent_session::domain::ports::REPLICA_HEARTBEAT_INTERVAL);
+ loop {
+ ticker.tick().await;
+ if let Err(error) = heartbeat_repo
+ .heartbeat(replica, heartbeat_address.as_ref())
+ .await
+ {
+ tracing::warn!(error = ?error, %replica, "failed to heartbeat harness replica");
+ }
+ }
+ });
+
// Keep trigger generation in this deployment while retaining Kafka as the
// boundary between channel events and harness commands.
let mut trigger = tokio::spawn(trigger::supervise(
@@ -679,6 +778,7 @@ async fn run() -> anyhow::Result<()> {
http.abort();
trigger.abort();
egress_http.abort();
+ heartbeat.abort();
let stop_failures = container_shutdown.shutdown_all().await;
if stop_failures > 0 {
tracing::error!(stop_failures, "some sandboxes failed to stop");