Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(agent_harness): route commands away from a draining replica (#7024)

Wolf Mermelsteineb3809104a16
diff --git a/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.json b/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.json
deleted file mode 100644
index e0fc201db3..0000000000
--- a/.sqlx/query-261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT live.id AS replica_id, live.address\n FROM agent_session\n JOIN harness_replica live ON live.id = agent_session.manager_replica_id\n WHERE agent_session.id = $1\n AND live.last_heartbeat_at > now() - make_interval(secs => $2)\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "replica_id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 1,
- "name": "address",
- "type_info": "Text"
- }
- ],
- "parameters": {
- "Left": [
- "Uuid",
- "Float8"
- ]
- },
- "nullable": [
- false,
- true
- ]
- },
- "hash": "261d5942ee454cde346794b4359e074c02c9336dc7c986b2580adf55fdf16e52"
-}
diff --git a/.sqlx/query-6a7c20eb5577db0bf0d2e48a979287b681a881ffc25a6151b227c6f7f0b493fd.json b/.sqlx/query-6a7c20eb5577db0bf0d2e48a979287b681a881ffc25a6151b227c6f7f0b493fd.json
new file mode 100644
index 0000000000..479b26424b
--- /dev/null
+++ b/.sqlx/query-6a7c20eb5577db0bf0d2e48a979287b681a881ffc25a6151b227c6f7f0b493fd.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO harness_replica (id, draining_at)\n VALUES ($1, now())\n ON CONFLICT (id) DO UPDATE\n SET draining_at = COALESCE(harness_replica.draining_at, now())\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "6a7c20eb5577db0bf0d2e48a979287b681a881ffc25a6151b227c6f7f0b493fd"
+}
diff --git a/.sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.json b/.sqlx/query-d14d902edcd7cc7f630777c955c874cf78cd17eadd87b0576fc74c3eaa334f78.json
similarity index 82%
rename from .sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.json
rename to .sqlx/query-d14d902edcd7cc7f630777c955c874cf78cd17eadd87b0576fc74c3eaa334f78.json
index 6dc5347586..53c5bb60ce 100644
--- a/.sqlx/query-1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd.json
+++ b/.sqlx/query-d14d902edcd7cc7f630777c955c874cf78cd17eadd87b0576fc74c3eaa334f78.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n WITH replica AS (\n INSERT INTO harness_replica (id, last_heartbeat_at)\n VALUES ($2, now())\n ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()\n )\n UPDATE agent_session\n SET manager_replica_id = $2,\n manager_fence = manager_fence + 1,\n modified_at = now()\n WHERE id = $1\n AND (\n manager_replica_id IS NULL\n OR manager_replica_id = $2\n OR NOT EXISTS (\n SELECT 1 FROM harness_replica live\n WHERE live.id = agent_session.manager_replica_id\n AND live.last_heartbeat_at > now() - make_interval(secs => $3)\n )\n )\n RETURNING manager_fence\n ",
+ "query": "\n WITH replica AS (\n INSERT INTO harness_replica (id, last_heartbeat_at)\n VALUES ($2, now())\n ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()\n )\n UPDATE agent_session\n SET manager_replica_id = $2,\n manager_fence = manager_fence + 1,\n modified_at = now()\n WHERE id = $1\n AND (\n manager_replica_id IS NULL\n OR manager_replica_id = $2\n OR NOT EXISTS (\n SELECT 1 FROM harness_replica live\n WHERE live.id = agent_session.manager_replica_id\n AND live.last_heartbeat_at > now() - make_interval(secs => $3)\n AND live.draining_at IS NULL\n )\n )\n RETURNING manager_fence\n ",
"describe": {
"columns": [
{
@@ -20,5 +20,5 @@
false
]
},
- "hash": "1a2b568193d4d16c9aa62cb7d8dfee75bdce4824f21561f2cf4b7c3390e422dd"
+ "hash": "d14d902edcd7cc7f630777c955c874cf78cd17eadd87b0576fc74c3eaa334f78"
}
diff --git a/.sqlx/query-fb950b8781c622a0a2ab6e1ecfcad83128686484103d19cb273cffcb08b50a07.json b/.sqlx/query-fb950b8781c622a0a2ab6e1ecfcad83128686484103d19cb273cffcb08b50a07.json
new file mode 100644
index 0000000000..47b4d9af43
--- /dev/null
+++ b/.sqlx/query-fb950b8781c622a0a2ab6e1ecfcad83128686484103d19cb273cffcb08b50a07.json
@@ -0,0 +1,42 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n live.id AS \"replica_id?\",\n live.address AS \"address?\",\n (live.draining_at IS NOT NULL) AS \"holder_draining?\",\n EXISTS (\n SELECT 1 FROM harness_replica me\n WHERE me.id = $3 AND me.draining_at IS NOT NULL\n ) AS \"asking_replica_draining!\"\n FROM (SELECT $1::uuid AS id) AS asked\n LEFT JOIN agent_session ON agent_session.id = asked.id\n LEFT JOIN harness_replica live\n ON live.id = agent_session.manager_replica_id\n AND live.last_heartbeat_at > now() - make_interval(secs => $2)\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "replica_id?",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "address?",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "holder_draining?",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 3,
+ "name": "asking_replica_draining!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Float8",
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ true,
+ true,
+ null,
+ null
+ ]
+ },
+ "hash": "fb950b8781c622a0a2ab6e1ecfcad83128686484103d19cb273cffcb08b50a07"
+}
diff --git a/crates/agent_harness/src/domain/service/queue.rs b/crates/agent_harness/src/domain/service/queue.rs
index 016d71902a..bfd1699acf 100644
--- a/crates/agent_harness/src/domain/service/queue.rs
+++ b/crates/agent_harness/src/domain/service/queue.rs
@@ -196,9 +196,10 @@ where
Notifier: AgentSessionNotifier,
{
/// Execute where the session's live actor is: locally when nobody (or
- /// this replica) manages it, on the managing peer otherwise.
+ /// this replica) manages it, on the managing peer otherwise, and nowhere
+ /// at all when this replica is draining.
/// The routing decision is recorded on the span, not only logged: which of
- /// the three answers the lease gave, which peer it named, and whether the
+ /// the lease's answers came back, which peer it named, and whether the
/// command left this process. Those are the fields you group by when a
/// replica is mishandling commands, and a log line cannot be aggregated.
#[tracing::instrument(
@@ -217,13 +218,21 @@ where
command: HarnessCommand,
) -> Result<CommandOutcome> {
let span = tracing::Span::current();
- // Open creates the row and has no existing manager to route through.
- if matches!(command, HarnessCommand::Open(_)) {
- span.record("agent.session.management", "open");
- span.record("agent.command.forwarded", false);
- return self.execute(session_id, command).await;
- }
let manager = match self.sessions.management(session_id).await? {
+ // First, an open included: a replica on its way out has no
+ // business taking on new work of any shape, and the caller's
+ // retry lands on one that is staying.
+ SessionManagement::Draining => {
+ span.record("agent.session.management", "draining");
+ span.record("agent.command.forwarded", false);
+ return Err(AgentSessionError::Draining(session_id).into());
+ }
+ // Open creates the row and has no existing manager to route through.
+ _ if matches!(command, HarnessCommand::Open(_)) => {
+ span.record("agent.session.management", "open");
+ span.record("agent.command.forwarded", false);
+ return self.execute(session_id, command).await;
+ }
SessionManagement::Unmanaged => {
span.record("agent.session.management", "unmanaged");
span.record("agent.command.forwarded", false);
diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rs
index 1c6a366bd0..ac28424752 100644
--- a/crates/agent_harness/src/domain/service/test.rs
+++ b/crates/agent_harness/src/domain/service/test.rs
@@ -5,6 +5,7 @@
mod chat_reply;
mod user_cleanup;
+use agent_session::domain::service::AgentSessionService as _;
use messages::domain::models::MessageParent;
use std::sync::{Arc, Mutex};
@@ -1224,6 +1225,56 @@ async fn forward_to_a_live_session_reuses_the_transport() {
assert_eq!(announced[1].origin_thread_id, Uuid::from_u128(0xf1));
}
+/// The failure this exists to stop: a rolling deploy's outgoing task keeps
+/// heartbeating for its whole drain window, so peers went on forwarding it
+/// prompts, it accepted them with a 200, and they died with the process
+/// seconds later. Once it has published its drain it starts nothing - not a
+/// prompt for a session it is still running, not an open for a new one - so
+/// the caller's retry lands on a replica that is staying.
+#[tokio::test]
+async fn a_draining_replica_starts_no_new_work() {
+ let ((service, _repo, containers, _announcer, _runtimes), mut turns) =
+ harness_with_signals(PromptContextMock::default(), PromptComposerMock::default());
+ let id = AgentSessionId::new();
+ let container = live_session(&service, &containers, id).await;
+ turns.settled(id).await;
+ let delivered = prompts(&container.agent()).len();
+
+ service
+ .inner
+ .sessions
+ .begin_draining()
+ .await
+ .expect("the drain is published");
+
+ let refused = service
+ .execute(
+ id,
+ HarnessCommand::Deliver(forward_message("how is it going")),
+ )
+ .await
+ .expect_err("a draining replica must not take a prompt it cannot finish");
+ assert!(
+ matches!(refused, HarnessError::Session(AgentSessionError::Draining(session)) if session == id),
+ "the refusal names the drain so the caller can retry, got {refused:?}"
+ );
+ assert_eq!(
+ prompts(&container.agent()).len(),
+ delivered,
+ "nothing reached the agent"
+ );
+
+ let refused_open = service
+ .execute(AgentSessionId::new(), HarnessCommand::Open(open_command()))
+ .await
+ .expect_err("a draining replica must not open a session either");
+ assert!(matches!(
+ refused_open,
+ HarnessError::Session(AgentSessionError::Draining(_))
+ ));
+ assert_eq!(containers.spawned(), 1, "no container for the refused open");
+}
+
#[tokio::test]
async fn composer_failure_stops_follow_up_announcement_and_delivery() {
let composer = PromptComposerMock::default();
diff --git a/crates/agent_session/src/domain/error.rs b/crates/agent_session/src/domain/error.rs
index 13caf62329..f68a687610 100644
--- a/crates/agent_session/src/domain/error.rs
+++ b/crates/agent_session/src/domain/error.rs
@@ -23,6 +23,11 @@ pub enum AgentSessionError {
AlreadyConnected(AgentSessionId),
#[error("agent session {0} is managed by another live replica")]
ManagedElsewhere(AgentSessionId),
+ /// This replica is shutting down, so it started nothing it could not
+ /// finish. Retryable: another replica is serving, and the retry lands
+ /// there.
+ #[error("this replica is draining; retry agent session {0} on another")]
+ Draining(AgentSessionId),
#[error("agent session {0} write was fenced out: another replica claimed the session")]
FencedOut(AgentSessionId),
#[error("acp handshake failed: {0}")]
diff --git a/crates/agent_session/src/domain/model.rs b/crates/agent_session/src/domain/model.rs
index 389e6cbc75..adb88b2145 100644
--- a/crates/agent_session/src/domain/model.rs
+++ b/crates/agent_session/src/domain/model.rs
@@ -87,6 +87,23 @@ pub struct SessionManager {
/// `None` means the manager is live but unreachable - hold the error
/// rather than execute somewhere the actor is not.
pub address: Option<ReplicaAddress>,
+ /// Whether the holder has published that it is shutting down. Still
+ /// heartbeating, so still "live" by the lease's own liveness rule, but
+ /// no longer somewhere to send work.
+ pub draining: bool,
+}
+
+/// What the lease says about a session, seen from one replica.
+#[derive(Debug, Clone)]
+pub struct LeaseView {
+ /// The replica holding the lease, when a live one holds it - draining
+ /// or not, because who may take over from a draining holder is the
+ /// reader's decision, not the store's.
+ pub holder: Option<SessionManager>,
+ /// Whether the replica that asked has published that it is draining.
+ /// Asked in the same statement as the holder: a command's routing turns
+ /// on both, and two round trips could straddle the drain.
+ pub asking_replica_draining: bool,
}
/// Where a session's live actor runs, from one service instance's viewpoint.
@@ -99,6 +116,9 @@ pub enum SessionManagement {
Ours,
/// A live peer manages it; commands belong at its address.
Peer(SessionManager),
+ /// This instance is draining: it is about to stop, so work sent here
+ /// would die with it. Commands belong on a replica that is staying.
+ Draining,
}
/// A session's takeover counter, bumped by every successful claim.
diff --git a/crates/agent_session/src/domain/ports.rs b/crates/agent_session/src/domain/ports.rs
index 15f2943840..c890bdbf9f 100644
--- a/crates/agent_session/src/domain/ports.rs
+++ b/crates/agent_session/src/domain/ports.rs
@@ -578,13 +578,24 @@ pub trait SessionOwnership: Send + Sync + 'static {
address: Option<&ReplicaAddress>,
) -> impl Future<Output = Result<()>> + Send;
- /// The live manager of a session, if a replica with a fresh heartbeat
- /// holds its lease. `None` covers both an unclaimed session and one whose
- /// holder has gone stale - either way the session is claimable.
- fn manager_of(
+ /// What the lease says about a session, from `replica`'s viewpoint: who
+ /// holds it (when a replica with a fresh heartbeat does - an absent
+ /// holder covers both an unclaimed session and one whose holder has gone
+ /// stale, either way claimable) and whether `replica` is itself draining.
+ fn lease_view(
&self,
session: AgentSessionId,
- ) -> impl Future<Output = Result<Option<SessionManager>>> + Send;
+ replica: ReplicaId,
+ ) -> impl Future<Output = Result<LeaseView>> + Send;
+
+ /// Publish that `replica` is shutting down, so nothing new is routed to
+ /// it while it drains.
+ ///
+ /// A deploy's old task keeps heartbeating for its whole drain window, so
+ /// liveness alone cannot tell a replica that is serving from one that is
+ /// leaving; this is the replica saying which it is. Idempotent: the first
+ /// drain wins, and a replica never un-drains.
+ fn begin_draining(&self, replica: ReplicaId) -> impl Future<Output = Result<()>> + Send;
}
#[cfg_attr(feature = "test-utils", mockall::automock)]
diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rs
index e3a6204fd1..2178a7a6bb 100644
--- a/crates/agent_session/src/domain/service.rs
+++ b/crates/agent_session/src/domain/service.rs
@@ -198,13 +198,23 @@ pub trait AgentSessionService: Send + Sync + 'static {
fn mark_disconnected(&self, id: AgentSessionId) -> impl Future<Output = Result<()>> + Send;
/// Where the session's live actor runs, from this instance's viewpoint:
- /// unmanaged (claimable here), ours, or a live peer's - in which case
- /// commands belong at the peer's address rather than in this process.
+ /// unmanaged (claimable here), ours, a live peer's - in which case
+ /// commands belong at the peer's address rather than in this process -
+ /// or nowhere worth sending work, because this instance is draining.
fn management(
&self,
id: AgentSessionId,
) -> impl Future<Output = Result<SessionManagement>> + Send;
+ /// Publish that this instance's replica is shutting down.
+ ///
+ /// Called the moment the process is told to stop, long before it
+ /// actually does: from here on peers route around this replica and it
+ /// routes work away from itself, so nothing new is started on a process
+ /// that will not live to finish it. Sessions already running here keep
+ /// running - they hold their claims until their actors wind down.
+ fn begin_draining(&self) -> impl Future<Output = Result<()>> + Send;
+
/// Attach a new transport to an existing persisted session.
///
/// The attachment carries the connection's handshake gate as well as the
@@ -804,13 +814,26 @@ where
}
async fn management(&self, id: AgentSessionId) -> Result<SessionManagement> {
- Ok(match self.repo.manager_of(id).await? {
+ let view = self.repo.lease_view(id, self.replica).await?;
+ if view.asking_replica_draining {
+ return Ok(SessionManagement::Draining);
+ }
+ Ok(match view.holder {
None => SessionManagement::Unmanaged,
+ // A draining holder is still heartbeating and may still be mid
+ // turn, but it is leaving: treated as claimable so the next
+ // command lands on a replica that is staying, and the fence the
+ // takeover bumps is what stops the two writing over each other.
+ Some(manager) if manager.draining => SessionManagement::Unmanaged,
Some(manager) if manager.replica == self.replica => SessionManagement::Ours,
Some(manager) => SessionManagement::Peer(manager),
})
}
+ async fn begin_draining(&self) -> Result<()> {
+ self.repo.begin_draining(self.replica).await
+ }
+
/// The out-of-band disconnect: a session marked dead by its opener rather
/// than by its own actor stopping. Spanned separately from
/// `agent.session.disconnect` because the two write the same log frame
diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rs
index 259ff30b36..bfb9c86fba 100644
--- a/crates/agent_session/src/domain/service/test.rs
+++ b/crates/agent_session/src/domain/service/test.rs
@@ -1,7 +1,7 @@
use super::*;
use crate::PROTOCOL_VERSION;
use crate::domain::model::{
- DEFAULT_AGENT_SESSION_NAME, Message, ReplicaAddress, SessionBot, SessionManager,
+ DEFAULT_AGENT_SESSION_NAME, LeaseView, Message, ReplicaAddress, SessionBot,
};
use crate::domain::ports::NoOpRealtime;
use crate::domain::ports::{NoOpTurnObserver, NoopLifecyclePublisher};
@@ -624,8 +624,12 @@ impl SessionOwnership for BlockingPromptLogs {
self.repo.heartbeat(replica, address).await
}
- async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {
- self.repo.manager_of(session).await
+ async fn lease_view(&self, session: AgentSessionId, replica: ReplicaId) -> Result<LeaseView> {
+ self.repo.lease_view(session, replica).await
+ }
+
+ async fn begin_draining(&self, replica: ReplicaId) -> Result<()> {
+ self.repo.begin_draining(replica).await
}
}
@@ -926,6 +930,61 @@ async fn a_second_replica_cannot_attach_a_session_with_a_live_manager() {
assert!(matches!(result, Err(AgentSessionError::ManagedElsewhere(id)) if id == fx.session));
}
+/// What a rolling deploy does to routing, from the service's side. The task
+/// being replaced keeps heartbeating for its whole drain window, so nothing
+/// about liveness stops work reaching it; the drain it publishes is what
+/// does. Both halves: the replica leaving reports itself as no place to send
+/// work, and the peer stops seeing it as the manager and takes the session
+/// over instead of waiting out a heartbeat that is still arriving.
+#[tokio::test]
+async fn a_draining_replica_stops_managing_its_sessions() {
+ let fx = fixture();
+ fx.service
+ .attach_session(fx.session, RuntimeAttachment::solo(PendingTransport))
+ .await
+ .expect("the first replica attaches");
+ let peer = AgentSessionServiceImpl::new(
+ fx.repo.clone(),
+ FoldedMessageService::new(fx.repo.clone()),
+ NoOpRealtime,
+ NoOpAgentSessionNameGenerator,
+ Arc::new(NoOpTurnObserver),
+ Arc::new(NoopLifecyclePublisher),
+ ReplicaId::mint(),
+ );
+ assert!(matches!(
+ fx.service.management(fx.session).await.expect("management"),
+ SessionManagement::Ours
+ ));
+ assert!(matches!(
+ peer.management(fx.session).await.expect("management"),
+ SessionManagement::Peer(manager) if manager.replica == fx.service.replica_id()
+ ));
+
+ fx.service
+ .begin_draining()
+ .await
+ .expect("the drain is published");
+
+ assert!(
+ matches!(
+ fx.service.management(fx.session).await.expect("management"),
+ SessionManagement::Draining
+ ),
+ "a replica on its way out sends work nowhere, its own sessions included"
+ );
+ assert!(
+ matches!(
+ peer.management(fx.session).await.expect("management"),
+ SessionManagement::Unmanaged
+ ),
+ "the holder is leaving, so the session is the staying replica's to take"
+ );
+ peer.attach_session(fx.session, RuntimeAttachment::solo(PendingTransport))
+ .await
+ .expect("the peer takes over from a draining holder");
+}
+
/// A command sent while the handshake never completes cannot hang its caller
/// forever - see [`HANDSHAKE_TIMEOUT`]. Without that bound, this test would
/// simply never finish. The actor it was stuck in cannot linger afterwards
diff --git a/crates/agent_session/src/inbound/axum_router.rs b/crates/agent_session/src/inbound/axum_router.rs
index a241692364..7c0a94fd6a 100644
--- a/crates/agent_session/src/inbound/axum_router.rs
+++ b/crates/agent_session/src/inbound/axum_router.rs
@@ -25,7 +25,7 @@ use agent_runtime_protocol::domain::{
use axum::{
Json, Router,
extract::{FromRef, Path, State},
- http::StatusCode,
+ http::{StatusCode, header},
response::{IntoResponse, Response},
routing::{delete, get, post, put},
};
@@ -342,6 +342,20 @@ impl IntoResponse for AgentSessionApiError {
};
(status, error.to_string()).into_response()
}
+ // This replica is on its way out of a rolling deploy. The work
+ // was not started, so the honest answer is "ask again" rather
+ // than a 200 for a command that would have died with the
+ // process: 503 with a retry hint, which is what the load
+ // balancer's next pick will serve.
+ Self::Domain(AgentSessionError::Draining(session_id)) => {
+ tracing::info!(%session_id, "command refused: this replica is draining");
+ (
+ StatusCode::SERVICE_UNAVAILABLE,
+ [(header::RETRY_AFTER, "1")],
+ "this replica is shutting down; retry",
+ )
+ .into_response()
+ }
// Already dispatched, removed, or never queued: the caller's
// entry is not waiting anymore, and there is no un-sending it.
Self::Domain(error @ AgentSessionError::QueuedControlNotFound) => {
diff --git a/crates/agent_session/src/outbound/postgres.rs b/crates/agent_session/src/outbound/postgres.rs
index 138e0d5675..c42178e769 100644
--- a/crates/agent_session/src/outbound/postgres.rs
+++ b/crates/agent_session/src/outbound/postgres.rs
@@ -18,9 +18,10 @@ mod working_branch;
use crate::domain::error::{AgentSessionError, Result};
use crate::domain::model::{
AgentMcpServers, AgentSession, AgentSessionId, AgentSessionLog, AgentSessionPreviewData,
- ClaimOutcome, CreateAgentSessionParams, ExternalSession, ManagerFence, Message, ReplicaAddress,
- ReplicaId, SandboxSize, SessionBot, SessionClaim, SessionManager, SessionPreviewCandidate,
- SessionStatus, StoredAgentSessionLog, StoredQueuedAction, ThreadSession, cursor_run_checkpoint,
+ ClaimOutcome, CreateAgentSessionParams, ExternalSession, LeaseView, ManagerFence, Message,
+ ReplicaAddress, ReplicaId, SandboxSize, SessionBot, SessionClaim, SessionManager,
+ SessionPreviewCandidate, SessionStatus, StoredAgentSessionLog, StoredQueuedAction,
+ ThreadSession, cursor_run_checkpoint,
};
use crate::domain::ports::{
AgentSessionLogRepo, AgentSessionRepo, ExternalSessionRepo, REPLICA_STALE_AFTER,
@@ -1452,6 +1453,7 @@ impl SessionOwnership for PgAgentSessionRepo {
SELECT 1 FROM harness_replica live
WHERE live.id = agent_session.manager_replica_id
AND live.last_heartbeat_at > now() - make_interval(secs => $3)
+ AND live.draining_at IS NULL
)
)
RETURNING manager_fence
@@ -1542,25 +1544,63 @@ impl SessionOwnership for PgAgentSessionRepo {
Ok(())
}
- async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {
+ async fn lease_view(&self, session: AgentSessionId, replica: ReplicaId) -> Result<LeaseView> {
+ // One statement for both halves: the holder comes from the session's
+ // lease, the asking replica's own drain from its heartbeat row, and
+ // reading them apart could straddle the moment this replica started
+ // draining and route a command to itself anyway. Built outward from
+ // the asked-for id rather than from `agent_session`, so a session
+ // that does not exist still answers the drain half.
let row = sqlx::query!(
r#"
- SELECT live.id AS replica_id, live.address
- FROM agent_session
- JOIN harness_replica live ON live.id = agent_session.manager_replica_id
- WHERE agent_session.id = $1
- AND live.last_heartbeat_at > now() - make_interval(secs => $2)
+ SELECT
+ live.id AS "replica_id?",
+ live.address AS "address?",
+ (live.draining_at IS NOT NULL) AS "holder_draining?",
+ EXISTS (
+ SELECT 1 FROM harness_replica me
+ WHERE me.id = $3 AND me.draining_at IS NOT NULL
+ ) AS "asking_replica_draining!"
+ FROM (SELECT $1::uuid AS id) AS asked
+ LEFT JOIN agent_session ON agent_session.id = asked.id
+ LEFT JOIN harness_replica live
+ ON live.id = agent_session.manager_replica_id
+ AND live.last_heartbeat_at > now() - make_interval(secs => $2)
"#,
session.as_uuid(),
REPLICA_STALE_AFTER.as_secs_f64(),
+ replica.as_uuid(),
)
- .fetch_optional(&self.pool)
+ .fetch_one(&self.pool)
.await
- .context("failed to read the agent session's live manager")?;
- Ok(row.map(|row| SessionManager {
- replica: ReplicaId::from_uuid(row.replica_id),
- address: row.address.map(ReplicaAddress::new),
- }))
+ .context("failed to read the agent session's lease")?;
+ Ok(LeaseView {
+ holder: row.replica_id.map(|replica_id| SessionManager {
+ replica: ReplicaId::from_uuid(replica_id),
+ address: row.address.map(ReplicaAddress::new),
+ draining: row.holder_draining.unwrap_or(false),
+ }),
+ asking_replica_draining: row.asking_replica_draining,
+ })
+ }
+
+ async fn begin_draining(&self, replica: ReplicaId) -> Result<()> {
+ // Upsert, not update: a replica that has not heartbeated yet still
+ // has to be able to say it is leaving, and the row it creates is
+ // stale from birth - which is exactly what it means.
+ sqlx::query!(
+ r#"
+ INSERT INTO harness_replica (id, draining_at)
+ VALUES ($1, now())
+ ON CONFLICT (id) DO UPDATE
+ SET draining_at = COALESCE(harness_replica.draining_at, now())
+ "#,
+ replica.as_uuid(),
+ )
+ .execute(&self.pool)
+ .await
+ .context("failed to publish that the harness replica is draining")?;
+ Ok(())
}
}
diff --git a/crates/agent_session/src/outbound/postgres/test.rs b/crates/agent_session/src/outbound/postgres/test.rs
index 11eb949778..96818fc4eb 100644
--- a/crates/agent_session/src/outbound/postgres/test.rs
+++ b/crates/agent_session/src/outbound/postgres/test.rs
@@ -1454,6 +1454,88 @@ async fn a_stale_holder_is_superseded(pool: PgPool) {
assert_eq!(takeover.fence, ManagerFence(2));
}
+/// A rolling deploy's outgoing task heartbeats normally for its whole drain
+/// window, so liveness alone keeps pointing every command at the replica that
+/// is leaving. Its published drain is what takes it out of the lease: peers
+/// stop reading it as the live manager and claim the session instead, without
+/// waiting out the heartbeat it is still sending.
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_draining_holder_stops_managing_and_can_be_taken_over(pool: PgPool) {
+ let repo = PgAgentSessionRepo::new(pool.clone());
+ let bot_id = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot_id, None, None)).await;
+ let leaving = ReplicaId::mint();
+ let staying = ReplicaId::mint();
+
+ claimed(repo.claim(session.id, leaving).await.expect("claim"));
+ repo.heartbeat(leaving, None).await.expect("fresh beat");
+ let before = repo
+ .lease_view(session.id, staying)
+ .await
+ .expect("read the lease");
+ assert_eq!(
+ before.holder.as_ref().map(|holder| holder.replica),
+ Some(leaving)
+ );
+ assert!(before.holder.is_some_and(|holder| !holder.draining));
+
+ repo.begin_draining(leaving).await.expect("publish drain");
+
+ let leaving_view = repo
+ .lease_view(session.id, leaving)
+ .await
+ .expect("read the lease");
+ assert!(
+ leaving_view.asking_replica_draining,
+ "the replica leaving can tell that it is"
+ );
+ assert!(
+ leaving_view.holder.is_some_and(|holder| holder.draining),
+ "it still holds the lease, marked as leaving"
+ );
+ let staying_view = repo
+ .lease_view(session.id, staying)
+ .await
+ .expect("read the lease");
+ assert!(!staying_view.asking_replica_draining);
+
+ let takeover = claimed(repo.claim(session.id, staying).await.expect("takeover"));
+ assert_eq!(takeover.fence, ManagerFence(2));
+}
+
+/// The drain is a one-way door, and a replica that never beat can still
+/// announce it: the row it creates is stale from birth, which is exactly what
+/// a process on its way out is.
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn draining_is_idempotent_and_needs_no_prior_heartbeat(pool: PgPool) {
+ let repo = PgAgentSessionRepo::new(pool.clone());
+ let bot_id = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot_id, None, None)).await;
+ let replica = ReplicaId::mint();
+
+ repo.begin_draining(replica).await.expect("first drain");
+ let first = drained_at(&pool, replica).await;
+ repo.begin_draining(replica).await.expect("second drain");
+ let second = drained_at(&pool, replica).await;
+ assert_eq!(first, second, "the first drain's moment is the one kept");
+ assert!(
+ repo.lease_view(session.id, replica)
+ .await
+ .expect("read the lease")
+ .asking_replica_draining
+ );
+}
+
+async fn drained_at(pool: &PgPool, replica: ReplicaId) -> Option<chrono::DateTime<chrono::Utc>> {
+ sqlx::query_scalar!(
+ r#"SELECT draining_at FROM harness_replica WHERE id = $1"#,
+ replica.as_uuid(),
+ )
+ .fetch_one(pool)
+ .await
+ .expect("the replica row exists")
+}
+
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
async fn release_frees_the_lease_but_never_a_successors(pool: PgPool) {
let repo = PgAgentSessionRepo::new(pool.clone());
diff --git a/crates/agent_session/src/testing.rs b/crates/agent_session/src/testing.rs
index 24a5489856..0f1bc3e71b 100644
--- a/crates/agent_session/src/testing.rs
+++ b/crates/agent_session/src/testing.rs
@@ -8,8 +8,8 @@ use crate::domain::error::{AgentSessionError, Result};
use crate::domain::events::AgentSessionLifecycleEvent;
use crate::domain::model::{
AgentMcpServers, AgentSession, AgentSessionId, AgentSessionLog, AgentSessionPreviewData,
- ClaimOutcome, CreateAgentSessionParams, DEFAULT_AGENT_SESSION_NAME, LogAppended, ManagerFence,
- ReplicaAddress, ReplicaId, SandboxSize, SessionBot, SessionClaim, SessionManager,
+ ClaimOutcome, CreateAgentSessionParams, DEFAULT_AGENT_SESSION_NAME, LeaseView, LogAppended,
+ ManagerFence, ReplicaAddress, ReplicaId, SandboxSize, SessionBot, SessionClaim, SessionManager,
SessionPreviewCandidate, SessionStatus, StoredAgentSessionLog, StoredQueuedAction,
ThreadSession,
};
@@ -34,7 +34,28 @@ mod working_branch;
type Lease = (Option<ReplicaId>, i64);
/// One replica's row: its last heartbeat and published forwarding address.
-type ReplicaRow = (std::time::Instant, Option<ReplicaAddress>);
+/// A replica's heartbeat row: when it last beat, where peers reach it, and
+/// whether it has said it is going away.
+#[derive(Debug, Clone)]
+struct ReplicaRow {
+ beat: std::time::Instant,
+ address: Option<ReplicaAddress>,
+ draining: bool,
+}
+
+impl ReplicaRow {
+ fn beating() -> Self {
+ Self {
+ beat: std::time::Instant::now(),
+ address: None,
+ draining: false,
+ }
+ }
+
+ fn live(&self) -> bool {
+ self.beat.elapsed() < REPLICA_STALE_AFTER
+ }
+}
/// An in-memory [`AgentSessionRepo`] and [`AgentSessionLogRepo`].
///
@@ -499,16 +520,19 @@ impl SessionOwnership for InMemoryAgentSessionRepo {
.replicas
.lock()
.expect("in-memory replica store is not poisoned");
- replicas.entry(replica).or_insert((now, None)).0 = now;
+ replicas
+ .entry(replica)
+ .or_insert_with(ReplicaRow::beating)
+ .beat = now;
let mut leases = self
.leases
.lock()
.expect("in-memory lease store is not poisoned");
let (holder, fence) = leases.entry(session).or_insert((None, 0));
let holder_is_live = holder.filter(|holder| *holder != replica).filter(|holder| {
- replicas
- .get(holder)
- .is_some_and(|(beat, _)| now.duration_since(*beat) < REPLICA_STALE_AFTER)
+ replicas.get(holder).is_some_and(|row| {
+ now.duration_since(row.beat) < REPLICA_STALE_AFTER && !row.draining
+ })
});
if let Some(holder) = holder_is_live {
return Ok(ClaimOutcome::ManagedElsewhere(holder));
@@ -541,37 +565,52 @@ impl SessionOwnership for InMemoryAgentSessionRepo {
.replicas
.lock()
.expect("in-memory replica store is not poisoned");
- let entry = replicas
- .entry(replica)
- .or_insert((std::time::Instant::now(), None));
- entry.0 = std::time::Instant::now();
+ let entry = replicas.entry(replica).or_insert_with(ReplicaRow::beating);
+ entry.beat = std::time::Instant::now();
// As in the real adapter: a beat carrying no address keeps the one
// already published.
if let Some(address) = address {
- entry.1 = Some(address.clone());
+ entry.address = Some(address.clone());
}
Ok(())
}
- async fn manager_of(&self, session: AgentSessionId) -> Result<Option<SessionManager>> {
+ async fn lease_view(&self, session: AgentSessionId, replica: ReplicaId) -> Result<LeaseView> {
let leases = self
.leases
.lock()
.expect("in-memory lease store is not poisoned");
- let Some((Some(holder), _)) = leases.get(&session) else {
- return Ok(None);
- };
let replicas = self
.replicas
.lock()
.expect("in-memory replica store is not poisoned");
- Ok(replicas
- .get(holder)
- .filter(|(beat, _)| beat.elapsed() < REPLICA_STALE_AFTER)
- .map(|(_, address)| SessionManager {
- replica: *holder,
- address: address.clone(),
- }))
+ let holder = match leases.get(&session) {
+ Some((Some(holder), _)) => {
+ replicas
+ .get(holder)
+ .filter(|row| row.live())
+ .map(|row| SessionManager {
+ replica: *holder,
+ address: row.address.clone(),
+ draining: row.draining,
+ })
+ }
+ _ => None,
+ };
+ Ok(LeaseView {
+ holder,
+ asking_replica_draining: replicas.get(&replica).is_some_and(|row| row.draining),
+ })
+ }
+
+ async fn begin_draining(&self, replica: ReplicaId) -> Result<()> {
+ self.replicas
+ .lock()
+ .expect("in-memory replica store is not poisoned")
+ .entry(replica)
+ .or_insert_with(ReplicaRow::beating)
+ .draining = true;
+ Ok(())
}
}
diff --git a/crates/macro_db_client/migrations/20260925222628_harness_replica_draining.sql b/crates/macro_db_client/migrations/20260925222628_harness_replica_draining.sql
new file mode 100644
index 0000000000..6736b948d2
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260925222628_harness_replica_draining.sql
@@ -0,0 +1,11 @@
+-- When a harness replica published that it is going away.
+--
+-- A replica's heartbeat answers "is it alive", which is not the question a
+-- router needs during a rolling deploy: a task keeps heartbeating normally
+-- for the whole drain window, so peers go on resolving it as a session's
+-- live manager and forwarding it commands it will not live to finish. This
+-- column is the replica saying so itself, the moment SIGTERM lands - NULL
+-- while it is serving, set once while it drains, never cleared (a restarted
+-- process is a new replica with a new row).
+ALTER TABLE harness_replica
+ ADD COLUMN draining_at TIMESTAMPTZ;
diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rs
index 0b03feb372..37a3721ec4 100644
--- a/services/agent_harness_service/src/main.rs
+++ b/services/agent_harness_service/src/main.rs
@@ -80,7 +80,7 @@ use agent_inmem::rig_engine::RigTurnEngine;
use agent_runtime_directory::PgAgentRuntimeDirectory;
use agent_session::domain::model::{AgentMcpServers, ReplicaId};
use agent_session::domain::ports::{NoOpRealtime, SessionOwnership as _};
-use agent_session::domain::service::AgentSessionServiceImpl;
+use agent_session::domain::service::{AgentSessionService, AgentSessionServiceImpl};
use agent_session::inbound::axum_router::{
AgentSessionControlState, AgentSessionRouterState, CreateSessionState,
};
@@ -848,6 +848,10 @@ async fn run() -> anyhow::Result<()> {
// the repository route below serves, so what the app offers is exactly
// what a session may select.
let open_repositories = Arc::clone(&reachable_repositories);
+ // Kept back from the move below so the drain announcement has something
+ // to speak through: it is the same service instance the harness routes
+ // commands with, and a clone shares its replica identity.
+ let draining_sessions = sessions.clone();
let harness = Arc::new(
AgentHarnessService::new(
sessions,
@@ -1132,6 +1136,22 @@ async fn run() -> anyhow::Result<()> {
tokio::select! {
() = &mut shutdown => {
tracing::info!("agent harness service shutting down");
+ // SIGTERM lands minutes before the process actually stops:
+ // ECS keeps the task draining while its heartbeat stays
+ // fresh, so peers go on resolving it as a session's live
+ // manager and forwarding it commands it will not live to
+ // finish - which is how a prompt reaches the harness, gets a
+ // 200, and is never seen again. Said here, before any
+ // teardown, so the work that follows goes to a replica that
+ // is staying.
+ match draining_sessions.begin_draining().await {
+ Ok(()) => tracing::info!(%replica, "harness replica is draining"),
+ Err(error) => tracing::error!(
+ error = ?error,
+ %replica,
+ "failed to publish the harness replica drain",
+ ),
+ }
break;
}
result = &mut trigger => {