Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(agent_harness): route commands away from a draining replica (#7024)
Wolf Mermelsteineb3809104a16
diff --git a/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.json b/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.jsondeleted file mode 100644index e0fc201db3..0000000000--- a/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.json+++ /dev/null@@ -1,29 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n SELECT live.id AS replica_id, live.address\n FROM agent_session\n JOIN harness_replica live ON live.id = agent_session.manager_replica_id\n WHERE agent_session.id = $1\n AND live.last_heartbeat_at > now() - make_interval(secs => $2)\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "replica_id",- "type_info": "Uuid"- },- {- "ordinal": 1,- "name": "address",- "type_info": "Text"- }- ],- "parameters": {- "Left": [- "Uuid",- "Float8"- ]- },- "nullable": [- false,- true- ]- },- "hash": "261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52"-}diff --git a/.sqlx/query-6a7c20eb5577db0bf0d2e48a979287b681a881ffc25a6151b227c6f7f0b493fd.json b/.sqlx/query-6a7c20eb5577db0bf0d2e48a979287b681a881ffc25a6151b227c6f7f0b493fd.jsonnew file mode 100644index 0000000000..479b26424b--- /dev/null+++ b/.sqlx/query-6a7c20eb5577db0bf0d2e48a979287b681a881ffc25a6151b227c6f7f0b493fd.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO harness_replica (id, draining_at)\n VALUES ($1, now())\n ON CONFLICT (id) DO UPDATE\n SET draining_at = COALESCE(harness_replica.draining_at, now())\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "6a7c20eb5577db0bf0d2e48a979287b681a881ffc25a6151b227c6f7f0b493fd"+}diff --git a/.sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.json b/.sqlx/query-d14d902edcd7cc7f630777c955c874cf78cd17eadd87b0576fc74c3eaa334f78.jsonsimilarity index 82%rename from .sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.jsonrename to .sqlx/query-d14d902edcd7cc7f630777c955c874cf78cd17eadd87b0576fc74c3eaa334f78.jsonindex 6dc5347586..53c5bb60ce 100644--- a/.sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.json+++ b/.sqlx/query-d14d902edcd7cc7f630777c955c874cf78cd17eadd87b0576fc74c3eaa334f78.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n WITH replica AS (\n INSERT INTO harness_replica (id, last_heartbeat_at)\n VALUES ($2, now())\n ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()\n )\n UPDATE agent_session\n SET manager_replica_id = $2,\n manager_fence = manager_fence + 1,\n modified_at = now()\n WHERE id = $1\n AND (\n manager_replica_id IS NULL\n OR manager_replica_id = $2\n OR NOT EXISTS (\n SELECT 1 FROM harness_replica live\n WHERE live.id = agent_session.manager_replica_id\n AND live.last_heartbeat_at > now() - make_interval(secs => $3)\n )\n )\n RETURNING manager_fence\n ",+ "query": "\n WITH replica AS (\n INSERT INTO harness_replica (id, last_heartbeat_at)\n VALUES ($2, now())\n ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()\n )\n UPDATE agent_session\n SET manager_replica_id = $2,\n manager_fence = manager_fence + 1,\n modified_at = now()\n WHERE id = $1\n AND (\n manager_replica_id IS NULL\n OR manager_replica_id = $2\n OR NOT EXISTS (\n SELECT 1 FROM harness_replica live\n WHERE live.id = agent_session.manager_replica_id\n AND live.last_heartbeat_at > now() - make_interval(secs => $3)\n AND live.draining_at IS NULL\n )\n )\n RETURNING manager_fence\n ","describe": {"columns": [{@@ -20,5 +20,5 @@false]},- "hash": "1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd"+ "hash": "d14d902edcd7cc7f630777c955c874cf78cd17eadd87b0576fc74c3eaa334f78"}diff --git a/.sqlx/query-fb950b8781c622a0a2ab6e1ecfcad83128686484103d19cb273cffcb08b50a07.json b/.sqlx/query-fb950b8781c622a0a2ab6e1ecfcad83128686484103d19cb273cffcb08b50a07.jsonnew file mode 100644index 0000000000..47b4d9af43--- /dev/null+++ b/.sqlx/query-fb950b8781c622a0a2ab6e1ecfcad83128686484103d19cb273cffcb08b50a07.json@@ -0,0 +1,42 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT\n live.id AS \"replica_id?\",\n live.address AS \"address?\",\n (live.draining_at IS NOT NULL) AS \"holder_draining?\",\n EXISTS (\n SELECT 1 FROM harness_replica me\n WHERE me.id = $3 AND me.draining_at IS NOT NULL\n ) AS \"asking_replica_draining!\"\n FROM (SELECT $1::uuid AS id) AS asked\n LEFT JOIN agent_session ON agent_session.id = asked.id\n LEFT JOIN harness_replica live\n ON live.id = agent_session.manager_replica_id\n AND live.last_heartbeat_at > now() - make_interval(secs => $2)\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "replica_id?",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "address?",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "holder_draining?",+ "type_info": "Bool"+ },+ {+ "ordinal": 3,+ "name": "asking_replica_draining!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Float8",+ "Uuid"+ ]+ },+ "nullable": [+ true,+ true,+ null,+ null+ ]+ },+ "hash": "fb950b8781c622a0a2ab6e1ecfcad83128686484103d19cb273cffcb08b50a07"+}diff --git a/crates/agent_harness/src/domain/service/queue.rs b/crates/agent_harness/src/domain/service/queue.rsindex 016d71902a..bfd1699acf 100644--- a/crates/agent_harness/src/domain/service/queue.rs+++ b/crates/agent_harness/src/domain/service/queue.rs@@ -196,9 +196,10 @@ whereNotifier: AgentSessionNotifier,{/// Execute where the session's live actor is: locally when nobody (or- /// this replica) manages it, on the managing peer otherwise.+ /// this replica) manages it, on the managing peer otherwise, and nowhere+ /// at all when this replica is draining./// The routing decision is recorded on the span, not only logged: which of- /// the three answers the lease gave, which peer it named, and whether the+ /// the lease's answers came back, which peer it named, and whether the/// command left this process. Those are the fields you group by when a/// replica is mishandling commands, and a log line cannot be aggregated.#[tracing::instrument(@@ -217,13 +218,21 @@ wherecommand: HarnessCommand,) -> Result<CommandOutcome> {let span = tracing::Span::current();- // Open creates the row and has no existing manager to route through.- if matches!(command, HarnessCommand::Open(_)) {- span.record("agent.session.management", "open");- span.record("agent.command.forwarded", false);- return self.execute(session_id, command).await;- }let manager = match self.sessions.management(session_id).await? {+ // First, an open included: a replica on its way out has no+ // business taking on new work of any shape, and the caller's+ // retry lands on one that is staying.+ SessionManagement::Draining => {+ span.record("agent.session.management", "draining");+ span.record("agent.command.forwarded", false);+ return Err(AgentSessionError::Draining(session_id).into());+ }+ // Open creates the row and has no existing manager to route through.+ _ if matches!(command, HarnessCommand::Open(_)) => {+ span.record("agent.session.management", "open");+ span.record("agent.command.forwarded", false);+ return self.execute(session_id, command).await;+ }SessionManagement::Unmanaged => {span.record("agent.session.management", "unmanaged");span.record("agent.command.forwarded", false);diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rsindex 1c6a366bd0..ac28424752 100644--- a/crates/agent_harness/src/domain/service/test.rs+++ b/crates/agent_harness/src/domain/service/test.rs@@ -5,6 +5,7 @@mod chat_reply;mod user_cleanup;+use agent_session::domain::service::AgentSessionService as _;use messages::domain::models::MessageParent;use std::sync::{Arc, Mutex};@@ -1224,6 +1225,56 @@ async fn forward_to_a_live_session_reuses_the_transport() {assert_eq!(announced[1].origin_thread_id, Uuid::from_u128(0xf1));}+/// The failure this exists to stop: a rolling deploy's outgoing task keeps+/// heartbeating for its whole drain window, so peers went on forwarding it+/// prompts, it accepted them with a 200, and they died with the process+/// seconds later. Once it has published its drain it starts nothing - not a+/// prompt for a session it is still running, not an open for a new one - so+/// the caller's retry lands on a replica that is staying.+#[tokio::test]+async fn a_draining_replica_starts_no_new_work() {+ let ((service, _repo, containers, _announcer, _runtimes), mut turns) =+ harness_with_signals(PromptContextMock::default(), PromptComposerMock::default());+ let id = AgentSessionId::new();+ let container = live_session(&service, &containers, id).await;+ turns.settled(id).await;+ let delivered = prompts(&container.agent()).len();++ service+ .inner+ .sessions+ .begin_draining()+ .await+ .expect("the drain is published");++ let refused = service+ .execute(+ id,+ HarnessCommand::Deliver(forward_message("how is it going")),+ )+ .await+ .expect_err("a draining replica must not take a prompt it cannot finish");+ assert!(+ matches!(refused, HarnessError::Session(AgentSessionError::Draining(session)) if session == id),+ "the refusal names the drain so the caller can retry, got {refused:?}"+ );+ assert_eq!(+ prompts(&container.agent()).len(),+ delivered,+ "nothing reached the agent"+ );++ let refused_open = service+ .execute(AgentSessionId::new(), HarnessCommand::Open(open_command()))+ .await+ .expect_err("a draining replica must not open a session either");+ assert!(matches!(+ refused_open,+ HarnessError::Session(AgentSessionError::Draining(_))+ ));+ assert_eq!(containers.spawned(), 1, "no container for the refused open");+}+#[tokio::test]async fn composer_failure_stops_follow_up_announcement_and_delivery() {let composer = PromptComposerMock::default();diff --git a/crates/agent_session/src/domain/error.rs b/crates/agent_session/src/domain/error.rsindex 13caf62329..f68a687610 100644--- a/crates/agent_session/src/domain/error.rs+++ b/crates/agent_session/src/domain/error.rs@@ -23,6 +23,11 @@ pub enum AgentSessionError {AlreadyConnected(AgentSessionId),#[error("agent session {0} is managed by another live replica")]ManagedElsewhere(AgentSessionId),+ /// This replica is shutting down, so it started nothing it could not+ /// finish. Retryable: another replica is serving, and the retry lands+ /// there.+ #[error("this replica is draining; retry agent session {0} on another")]+ Draining(AgentSessionId),#[error("agent session {0} write was fenced out: another replica claimed the session")]FencedOut(AgentSessionId),#[error("acp handshake failed: {0}")]diff --git a/crates/agent_session/src/domain/model.rs b/crates/agent_session/src/domain/model.rsindex 389e6cbc75..adb88b2145 100644--- a/crates/agent_session/src/domain/model.rs+++ b/crates/agent_session/src/domain/model.rs@@ -87,6 +87,23 @@ pub struct SessionManager {/// `None` means the manager is live but unreachable - hold the error/// rather than execute somewhere the actor is not.pub address: Option<ReplicaAddress>,+ /// Whether the holder has published that it is shutting down. Still+ /// heartbeating, so still "live" by the lease's own liveness rule, but+ /// no longer somewhere to send work.+ pub draining: bool,+}++/// What the lease says about a session, seen from one replica.+#[derive(Debug, Clone)]+pub struct LeaseView {+ /// The replica holding the lease, when a live one holds it - draining+ /// or not, because who may take over from a draining holder is the+ /// reader's decision, not the store's.+ pub holder: Option<SessionManager>,+ /// Whether the replica that asked has published that it is draining.+ /// Asked in the same statement as the holder: a command's routing turns+ /// on both, and two round trips could straddle the drain.+ pub asking_replica_draining: bool,}/// Where a session's live actor runs, from one service instance's viewpoint.@@ -99,6 +116,9 @@ pub enum SessionManagement {Ours,/// A live peer manages it; commands belong at its address.Peer(SessionManager),+ /// This instance is draining: it is about to stop, so work sent here+ /// would die with it. Commands belong on a replica that is staying.+ Draining,}/// A session's takeover counter, bumped by every successful claim.diff --git a/crates/agent_session/src/domain/ports.rs b/crates/agent_session/src/domain/ports.rsindex 15f2943840..c890bdbf9f 100644--- a/crates/agent_session/src/domain/ports.rs+++ b/crates/agent_session/src/domain/ports.rs@@ -578,13 +578,24 @@ pub trait SessionOwnership: Send + Sync + 'static {address: Option<&ReplicaAddress>,) -> impl Future<Output = Result<()>> + Send;- /// The live manager of a session, if a replica with a fresh heartbeat- /// holds its lease. `None` covers both an unclaimed session and one whose- /// holder has gone stale - either way the session is claimable.- fn manager_of(+ /// What the lease says about a session, from `replica`'s viewpoint: who+ /// holds it (when a replica with a fresh heartbeat does - an absent+ /// holder covers both an unclaimed session and one whose holder has gone+ /// stale, either way claimable) and whether `replica` is itself draining.+ fn lease_view(&self,session: AgentSessionId,- ) -> impl Future<Output = Result<Option<SessionManager>>> + Send;+ replica: ReplicaId,+ ) -> impl Future<Output = Result<LeaseView>> + Send;++ /// Publish that `replica` is shutting down, so nothing new is routed to+ /// it while it drains.+ ///+ /// A deploy's old task keeps heartbeating for its whole drain window, so+ /// liveness alone cannot tell a replica that is serving from one that is+ /// leaving; this is the replica saying which it is. Idempotent: the first+ /// drain wins, and a replica never un-drains.+ fn begin_draining(&self, replica: ReplicaId) -> impl Future<Output = Result<()>> + Send;}#[cfg_attr(feature = "test-utils", mockall::automock)]diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rsindex e3a6204fd1..2178a7a6bb 100644--- a/crates/agent_session/src/domain/service.rs+++ b/crates/agent_session/src/domain/service.rs@@ -198,13 +198,23 @@ pub trait AgentSessionService: Send + Sync + 'static {fn mark_disconnected(&self, id: AgentSessionId) -> impl Future<Output = Result<()>> + Send;/// Where the session's live actor runs, from this instance's viewpoint:- /// unmanaged (claimable here), ours, or a live peer's - in which case- /// commands belong at the peer's address rather than in this process.+ /// unmanaged (claimable here), ours, a live peer's - in which case+ /// commands belong at the peer's address rather than in this process -+ /// or nowhere worth sending work, because this instance is draining.fn management(&self,id: AgentSessionId,) -> impl Future<Output = Result<SessionManagement>> + Send;+ /// Publish that this instance's replica is shutting down.+ ///+ /// Called the moment the process is told to stop, long before it+ /// actually does: from here on peers route around this replica and it+ /// routes work away from itself, so nothing new is started on a process+ /// that will not live to finish it. Sessions already running here keep+ /// running - they hold their claims until their actors wind down.+ fn begin_draining(&self) -> impl Future<Output = Result<()>> + Send;+/// Attach a new transport to an existing persisted session.////// The attachment carries the connection's handshake gate as well as the@@ -804,13 +814,26 @@ where}async fn management(&self, id: AgentSessionId) -> Result<SessionManagement> {- Ok(match self.repo.manager_of(id).await? {+ let view = self.repo.lease_view(id, self.replica).await?;+ if view.asking_replica_draining {+ return Ok(SessionManagement::Draining);+ }+ Ok(match view.holder {None => SessionManagement::Unmanaged,+ // A draining holder is still heartbeating and may still be mid+ // turn, but it is leaving: treated as claimable so the next+ // command lands on a replica that is staying, and the fence the+ // takeover bumps is what stops the two writing over each other.+ Some(manager) if manager.draining => SessionManagement::Unmanaged,Some(manager) if manager.replica == self.replica => SessionManagement::Ours,Some(manager) => SessionManagement::Peer(manager),})}+ async fn begin_draining(&self) -> Result<()> {+ self.repo.begin_draining(self.replica).await+ }+/// The out-of-band disconnect: a session marked dead by its opener rather/// than by its own actor stopping. Spanned separately from/// `agent.session.disconnect` because the two write the same log framediff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rsindex 259ff30b36..bfb9c86fba 100644--- a/crates/agent_session/src/domain/service/test.rs+++ b/crates/agent_session/src/domain/service/test.rs@@ -1,7 +1,7 @@use super::*;use crate::PROTOCOL_VERSION;use crate::domain::model::{- DEFAULT_AGENT_SESSION_NAME, Message, ReplicaAddress, SessionBot, SessionManager,+ DEFAULT_AGENT_SESSION_NAME, LeaseView, Message, ReplicaAddress, SessionBot,};use crate::domain::ports::NoOpRealtime;use crate::domain::ports::{NoOpTurnObserver, NoopLifecyclePublisher};@@ -624,8 +624,12 @@ impl SessionOwnership for BlockingPromptLogs {self.repo.heartbeat(replica, address).await}- async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {- self.repo.manager_of(session).await+ async fn lease_view(&self, session: AgentSessionId, replica: ReplicaId) -> Result<LeaseView> {+ self.repo.lease_view(session, replica).await+ }++ async fn begin_draining(&self, replica: ReplicaId) -> Result<()> {+ self.repo.begin_draining(replica).await}}@@ -926,6 +930,61 @@ async fn a_second_replica_cannot_attach_a_session_with_a_live_manager() {assert!(matches!(result, Err(AgentSessionError::ManagedElsewhere(id)) if id == fx.session));}+/// What a rolling deploy does to routing, from the service's side. The task+/// being replaced keeps heartbeating for its whole drain window, so nothing+/// about liveness stops work reaching it; the drain it publishes is what+/// does. Both halves: the replica leaving reports itself as no place to send+/// work, and the peer stops seeing it as the manager and takes the session+/// over instead of waiting out a heartbeat that is still arriving.+#[tokio::test]+async fn a_draining_replica_stops_managing_its_sessions() {+ let fx = fixture();+ fx.service+ .attach_session(fx.session, RuntimeAttachment::solo(PendingTransport))+ .await+ .expect("the first replica attaches");+ let peer = AgentSessionServiceImpl::new(+ fx.repo.clone(),+ FoldedMessageService::new(fx.repo.clone()),+ NoOpRealtime,+ NoOpAgentSessionNameGenerator,+ Arc::new(NoOpTurnObserver),+ Arc::new(NoopLifecyclePublisher),+ ReplicaId::mint(),+ );+ assert!(matches!(+ fx.service.management(fx.session).await.expect("management"),+ SessionManagement::Ours+ ));+ assert!(matches!(+ peer.management(fx.session).await.expect("management"),+ SessionManagement::Peer(manager) if manager.replica == fx.service.replica_id()+ ));++ fx.service+ .begin_draining()+ .await+ .expect("the drain is published");++ assert!(+ matches!(+ fx.service.management(fx.session).await.expect("management"),+ SessionManagement::Draining+ ),+ "a replica on its way out sends work nowhere, its own sessions included"+ );+ assert!(+ matches!(+ peer.management(fx.session).await.expect("management"),+ SessionManagement::Unmanaged+ ),+ "the holder is leaving, so the session is the staying replica's to take"+ );+ peer.attach_session(fx.session, RuntimeAttachment::solo(PendingTransport))+ .await+ .expect("the peer takes over from a draining holder");+}+/// A command sent while the handshake never completes cannot hang its caller/// forever - see [`HANDSHAKE_TIMEOUT`]. Without that bound, this test would/// simply never finish. The actor it was stuck in cannot linger afterwardsdiff --git a/crates/agent_session/src/inbound/axum_router.rs b/crates/agent_session/src/inbound/axum_router.rsindex a241692364..7c0a94fd6a 100644--- a/crates/agent_session/src/inbound/axum_router.rs+++ b/crates/agent_session/src/inbound/axum_router.rs@@ -25,7 +25,7 @@ use agent_runtime_protocol::domain::{use axum::{Json, Router,extract::{FromRef, Path, State},- http::StatusCode,+ http::{StatusCode, header},response::{IntoResponse, Response},routing::{delete, get, post, put},};@@ -342,6 +342,20 @@ impl IntoResponse for AgentSessionApiError {};(status, error.to_string()).into_response()}+ // This replica is on its way out of a rolling deploy. The work+ // was not started, so the honest answer is "ask again" rather+ // than a 200 for a command that would have died with the+ // process: 503 with a retry hint, which is what the load+ // balancer's next pick will serve.+ Self::Domain(AgentSessionError::Draining(session_id)) => {+ tracing::info!(%session_id, "command refused: this replica is draining");+ (+ StatusCode::SERVICE_UNAVAILABLE,+ [(header::RETRY_AFTER, "1")],+ "this replica is shutting down; retry",+ )+ .into_response()+ }// Already dispatched, removed, or never queued: the caller's// entry is not waiting anymore, and there is no un-sending it.Self::Domain(error @ AgentSessionError::QueuedControlNotFound) => {diff --git a/crates/agent_session/src/outbound/postgres.rs b/crates/agent_session/src/outbound/postgres.rsindex 138e0d5675..c42178e769 100644--- a/crates/agent_session/src/outbound/postgres.rs+++ b/crates/agent_session/src/outbound/postgres.rs@@ -18,9 +18,10 @@ mod working_branch;use crate::domain::error::{AgentSessionError, Result};use crate::domain::model::{AgentMcpServers, AgentSession, AgentSessionId, AgentSessionLog, AgentSessionPreviewData,- ClaimOutcome, CreateAgentSessionParams, ExternalSession, ManagerFence, Message, ReplicaAddress,- ReplicaId, SandboxSize, SessionBot, SessionClaim, SessionManager, SessionPreviewCandidate,- SessionStatus, StoredAgentSessionLog, StoredQueuedAction, ThreadSession, cursor_run_checkpoint,+ ClaimOutcome, CreateAgentSessionParams, ExternalSession, LeaseView, ManagerFence, Message,+ ReplicaAddress, ReplicaId, SandboxSize, SessionBot, SessionClaim, SessionManager,+ SessionPreviewCandidate, SessionStatus, StoredAgentSessionLog, StoredQueuedAction,+ ThreadSession, cursor_run_checkpoint,};use crate::domain::ports::{AgentSessionLogRepo, AgentSessionRepo, ExternalSessionRepo, REPLICA_STALE_AFTER,@@ -1452,6 +1453,7 @@ impl SessionOwnership for PgAgentSessionRepo {SELECT 1 FROM harness_replica liveWHERE live.id = agent_session.manager_replica_idAND live.last_heartbeat_at > now() - make_interval(secs => $3)+ AND live.draining_at IS NULL))RETURNING manager_fence@@ -1542,25 +1544,63 @@ impl SessionOwnership for PgAgentSessionRepo {Ok(())}- async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {+ async fn lease_view(&self, session: AgentSessionId, replica: ReplicaId) -> Result<LeaseView> {+ // One statement for both halves: the holder comes from the session's+ // lease, the asking replica's own drain from its heartbeat row, and+ // reading them apart could straddle the moment this replica started+ // draining and route a command to itself anyway. Built outward from+ // the asked-for id rather than from `agent_session`, so a session+ // that does not exist still answers the drain half.let row = sqlx::query!(r#"- SELECT live.id AS replica_id, live.address- FROM agent_session- JOIN harness_replica live ON live.id = agent_session.manager_replica_id- WHERE agent_session.id = $1- AND live.last_heartbeat_at > now() - make_interval(secs => $2)+ SELECT+ live.id AS "replica_id?",+ live.address AS "address?",+ (live.draining_at IS NOT NULL) AS "holder_draining?",+ EXISTS (+ SELECT 1 FROM harness_replica me+ WHERE me.id = $3 AND me.draining_at IS NOT NULL+ ) AS "asking_replica_draining!"+ FROM (SELECT $1::uuid AS id) AS asked+ LEFT JOIN agent_session ON agent_session.id = asked.id+ LEFT JOIN harness_replica live+ ON live.id = agent_session.manager_replica_id+ AND live.last_heartbeat_at > now() - make_interval(secs => $2)"#,session.as_uuid(),REPLICA_STALE_AFTER.as_secs_f64(),+ replica.as_uuid(),)- .fetch_optional(&self.pool)+ .fetch_one(&self.pool).await- .context("failed to read the agent session's live manager")?;- Ok(row.map(|row| SessionManager {- replica: ReplicaId::from_uuid(row.replica_id),- address: row.address.map(ReplicaAddress::new),- }))+ .context("failed to read the agent session's lease")?;+ Ok(LeaseView {+ holder: row.replica_id.map(|replica_id| SessionManager {+ replica: ReplicaId::from_uuid(replica_id),+ address: row.address.map(ReplicaAddress::new),+ draining: row.holder_draining.unwrap_or(false),+ }),+ asking_replica_draining: row.asking_replica_draining,+ })+ }++ async fn begin_draining(&self, replica: ReplicaId) -> Result<()> {+ // Upsert, not update: a replica that has not heartbeated yet still+ // has to be able to say it is leaving, and the row it creates is+ // stale from birth - which is exactly what it means.+ sqlx::query!(+ r#"+ INSERT INTO harness_replica (id, draining_at)+ VALUES ($1, now())+ ON CONFLICT (id) DO UPDATE+ SET draining_at = COALESCE(harness_replica.draining_at, now())+ "#,+ replica.as_uuid(),+ )+ .execute(&self.pool)+ .await+ .context("failed to publish that the harness replica is draining")?;+ Ok(())}}diff --git a/crates/agent_session/src/outbound/postgres/test.rs b/crates/agent_session/src/outbound/postgres/test.rsindex 11eb949778..96818fc4eb 100644--- a/crates/agent_session/src/outbound/postgres/test.rs+++ b/crates/agent_session/src/outbound/postgres/test.rs@@ -1454,6 +1454,88 @@ async fn a_stale_holder_is_superseded(pool: PgPool) {assert_eq!(takeover.fence, ManagerFence(2));}+/// A rolling deploy's outgoing task heartbeats normally for its whole drain+/// window, so liveness alone keeps pointing every command at the replica that+/// is leaving. Its published drain is what takes it out of the lease: peers+/// stop reading it as the live manager and claim the session instead, without+/// waiting out the heartbeat it is still sending.+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_draining_holder_stops_managing_and_can_be_taken_over(pool: PgPool) {+ let repo = PgAgentSessionRepo::new(pool.clone());+ let bot_id = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot_id, None, None)).await;+ let leaving = ReplicaId::mint();+ let staying = ReplicaId::mint();++ claimed(repo.claim(session.id, leaving).await.expect("claim"));+ repo.heartbeat(leaving, None).await.expect("fresh beat");+ let before = repo+ .lease_view(session.id, staying)+ .await+ .expect("read the lease");+ assert_eq!(+ before.holder.as_ref().map(|holder| holder.replica),+ Some(leaving)+ );+ assert!(before.holder.is_some_and(|holder| !holder.draining));++ repo.begin_draining(leaving).await.expect("publish drain");++ let leaving_view = repo+ .lease_view(session.id, leaving)+ .await+ .expect("read the lease");+ assert!(+ leaving_view.asking_replica_draining,+ "the replica leaving can tell that it is"+ );+ assert!(+ leaving_view.holder.is_some_and(|holder| holder.draining),+ "it still holds the lease, marked as leaving"+ );+ let staying_view = repo+ .lease_view(session.id, staying)+ .await+ .expect("read the lease");+ assert!(!staying_view.asking_replica_draining);++ let takeover = claimed(repo.claim(session.id, staying).await.expect("takeover"));+ assert_eq!(takeover.fence, ManagerFence(2));+}++/// The drain is a one-way door, and a replica that never beat can still+/// announce it: the row it creates is stale from birth, which is exactly what+/// a process on its way out is.+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn draining_is_idempotent_and_needs_no_prior_heartbeat(pool: PgPool) {+ let repo = PgAgentSessionRepo::new(pool.clone());+ let bot_id = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot_id, None, None)).await;+ let replica = ReplicaId::mint();++ repo.begin_draining(replica).await.expect("first drain");+ let first = drained_at(&pool, replica).await;+ repo.begin_draining(replica).await.expect("second drain");+ let second = drained_at(&pool, replica).await;+ assert_eq!(first, second, "the first drain's moment is the one kept");+ assert!(+ repo.lease_view(session.id, replica)+ .await+ .expect("read the lease")+ .asking_replica_draining+ );+}++async fn drained_at(pool: &PgPool, replica: ReplicaId) -> Option<chrono::DateTime<chrono::Utc>> {+ sqlx::query_scalar!(+ r#"SELECT draining_at FROM harness_replica WHERE id = $1"#,+ replica.as_uuid(),+ )+ .fetch_one(pool)+ .await+ .expect("the replica row exists")+}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn release_frees_the_lease_but_never_a_successors(pool: PgPool) {let repo = PgAgentSessionRepo::new(pool.clone());diff --git a/crates/agent_session/src/testing.rs b/crates/agent_session/src/testing.rsindex 24a5489856..0f1bc3e71b 100644--- a/crates/agent_session/src/testing.rs+++ b/crates/agent_session/src/testing.rs@@ -8,8 +8,8 @@ use crate::domain::error::{AgentSessionError, Result};use crate::domain::events::AgentSessionLifecycleEvent;use crate::domain::model::{AgentMcpServers, AgentSession, AgentSessionId, AgentSessionLog, AgentSessionPreviewData,- ClaimOutcome, CreateAgentSessionParams, DEFAULT_AGENT_SESSION_NAME, LogAppended, ManagerFence,- ReplicaAddress, ReplicaId, SandboxSize, SessionBot, SessionClaim, SessionManager,+ ClaimOutcome, CreateAgentSessionParams, DEFAULT_AGENT_SESSION_NAME, LeaseView, LogAppended,+ ManagerFence, ReplicaAddress, ReplicaId, SandboxSize, SessionBot, SessionClaim, SessionManager,SessionPreviewCandidate, SessionStatus, StoredAgentSessionLog, StoredQueuedAction,ThreadSession,};@@ -34,7 +34,28 @@ mod working_branch;type Lease = (Option<ReplicaId>, i64);/// One replica's row: its last heartbeat and published forwarding address.-type ReplicaRow = (std::time::Instant, Option<ReplicaAddress>);+/// A replica's heartbeat row: when it last beat, where peers reach it, and+/// whether it has said it is going away.+#[derive(Debug, Clone)]+struct ReplicaRow {+ beat: std::time::Instant,+ address: Option<ReplicaAddress>,+ draining: bool,+}++impl ReplicaRow {+ fn beating() -> Self {+ Self {+ beat: std::time::Instant::now(),+ address: None,+ draining: false,+ }+ }++ fn live(&self) -> bool {+ self.beat.elapsed() < REPLICA_STALE_AFTER+ }+}/// An in-memory [`AgentSessionRepo`] and [`AgentSessionLogRepo`].///@@ -499,16 +520,19 @@ impl SessionOwnership for InMemoryAgentSessionRepo {.replicas.lock().expect("in-memory replica store is not poisoned");- replicas.entry(replica).or_insert((now, None)).0 = now;+ replicas+ .entry(replica)+ .or_insert_with(ReplicaRow::beating)+ .beat = now;let mut leases = self.leases.lock().expect("in-memory lease store is not poisoned");let (holder, fence) = leases.entry(session).or_insert((None, 0));let holder_is_live = holder.filter(|holder| *holder != replica).filter(|holder| {- replicas- .get(holder)- .is_some_and(|(beat, _)| now.duration_since(*beat) < REPLICA_STALE_AFTER)+ replicas.get(holder).is_some_and(|row| {+ now.duration_since(row.beat) < REPLICA_STALE_AFTER && !row.draining+ })});if let Some(holder) = holder_is_live {return Ok(ClaimOutcome::ManagedElsewhere(holder));@@ -541,37 +565,52 @@ impl SessionOwnership for InMemoryAgentSessionRepo {.replicas.lock().expect("in-memory replica store is not poisoned");- let entry = replicas- .entry(replica)- .or_insert((std::time::Instant::now(), None));- entry.0 = std::time::Instant::now();+ let entry = replicas.entry(replica).or_insert_with(ReplicaRow::beating);+ entry.beat = std::time::Instant::now();// As in the real adapter: a beat carrying no address keeps the one// already published.if let Some(address) = address {- entry.1 = Some(address.clone());+ entry.address = Some(address.clone());}Ok(())}- async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {+ async fn lease_view(&self, session: AgentSessionId, replica: ReplicaId) -> Result<LeaseView> {let leases = self.leases.lock().expect("in-memory lease store is not poisoned");- let Some((Some(holder), _)) = leases.get(&session) else {- return Ok(None);- };let replicas = self.replicas.lock().expect("in-memory replica store is not poisoned");- Ok(replicas- .get(holder)- .filter(|(beat, _)| beat.elapsed() < REPLICA_STALE_AFTER)- .map(|(_, address)| SessionManager {- replica: *holder,- address: address.clone(),- }))+ let holder = match leases.get(&session) {+ Some((Some(holder), _)) => {+ replicas+ .get(holder)+ .filter(|row| row.live())+ .map(|row| SessionManager {+ replica: *holder,+ address: row.address.clone(),+ draining: row.draining,+ })+ }+ _ => None,+ };+ Ok(LeaseView {+ holder,+ asking_replica_draining: replicas.get(&replica).is_some_and(|row| row.draining),+ })+ }++ async fn begin_draining(&self, replica: ReplicaId) -> Result<()> {+ self.replicas+ .lock()+ .expect("in-memory replica store is not poisoned")+ .entry(replica)+ .or_insert_with(ReplicaRow::beating)+ .draining = true;+ Ok(())}}diff --git a/crates/macro_db_client/migrations/20260925222628_harness_replica_draining.sql b/crates/macro_db_client/migrations/20260925222628_harness_replica_draining.sqlnew file mode 100644index 0000000000..6736b948d2--- /dev/null+++ b/crates/macro_db_client/migrations/20260925222628_harness_replica_draining.sql@@ -0,0 +1,11 @@+-- When a harness replica published that it is going away.+--+-- A replica's heartbeat answers "is it alive", which is not the question a+-- router needs during a rolling deploy: a task keeps heartbeating normally+-- for the whole drain window, so peers go on resolving it as a session's+-- live manager and forwarding it commands it will not live to finish. This+-- column is the replica saying so itself, the moment SIGTERM lands - NULL+-- while it is serving, set once while it drains, never cleared (a restarted+-- process is a new replica with a new row).+ALTER TABLE harness_replica+ ADD COLUMN draining_at TIMESTAMPTZ;diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rsindex 0b03feb372..37a3721ec4 100644--- a/services/agent_harness_service/src/main.rs+++ b/services/agent_harness_service/src/main.rs@@ -80,7 +80,7 @@ use agent_inmem::rig_engine::RigTurnEngine;use agent_runtime_directory::PgAgentRuntimeDirectory;use agent_session::domain::model::{AgentMcpServers, ReplicaId};use agent_session::domain::ports::{NoOpRealtime, SessionOwnership as _};-use agent_session::domain::service::AgentSessionServiceImpl;+use agent_session::domain::service::{AgentSessionService, AgentSessionServiceImpl};use agent_session::inbound::axum_router::{AgentSessionControlState, AgentSessionRouterState, CreateSessionState,};@@ -848,6 +848,10 @@ async fn run() -> anyhow::Result<()> {// the repository route below serves, so what the app offers is exactly// what a session may select.let open_repositories = Arc::clone(&reachable_repositories);+ // Kept back from the move below so the drain announcement has something+ // to speak through: it is the same service instance the harness routes+ // commands with, and a clone shares its replica identity.+ let draining_sessions = sessions.clone();let harness = Arc::new(AgentHarnessService::new(sessions,@@ -1132,6 +1136,22 @@ async fn run() -> anyhow::Result<()> {tokio::select! {() = &mut shutdown => {tracing::info!("agent harness service shutting down");+ // SIGTERM lands minutes before the process actually stops:+ // ECS keeps the task draining while its heartbeat stays+ // fresh, so peers go on resolving it as a session's live+ // manager and forwarding it commands it will not live to+ // finish - which is how a prompt reaches the harness, gets a+ // 200, and is never seen again. Said here, before any+ // teardown, so the work that follows goes to a replica that+ // is staying.+ match draining_sessions.begin_draining().await {+ Ok(()) => tracing::info!(%replica, "harness replica is draining"),+ Err(error) => tracing::error!(+ error = ?error,+ %replica,+ "failed to publish the harness replica drain",+ ),+ }break;}result = &mut trigger => {