Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(harnesses): stop a stale socket close marking a live harness offline (#9872)
Eric Hayes776ca7854bc2
diff --git a/.sqlx/query-d30ffc9d1a8aabb56674f4a0292176684576a144a1dd05b4e6fed48f4e49e3a9.json b/.sqlx/query-1c08845abe8781e7beb18b8dbf5e367cf3b95acc7a26029a553efd8e2ca2b7a9.jsonsimilarity index 60%rename from .sqlx/query-d30ffc9d1a8aabb56674f4a0292176684576a144a1dd05b4e6fed48f4e49e3a9.jsonrename to .sqlx/query-1c08845abe8781e7beb18b8dbf5e367cf3b95acc7a26029a553efd8e2ca2b7a9.jsonindex ad20fc9231..1036e6b7be 100644--- a/.sqlx/query-d30ffc9d1a8aabb56674f4a0292176684576a144a1dd05b4e6fed48f4e49e3a9.json+++ b/.sqlx/query-1c08845abe8781e7beb18b8dbf5e367cf3b95acc7a26029a553efd8e2ca2b7a9.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n SELECT\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at\n FROM harnesses\n WHERE deleted_at IS NULL\n AND (\n owner_user_id = $1\n OR team_id IN (SELECT team_id FROM team_user WHERE user_id = $1)\n )\n ORDER BY created_at DESC\n ",+ "query": "\n SELECT\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at,\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()\n ) AS \"has_live_connection!\",\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id\n ) AS \"has_connection_record!\"\n FROM harnesses\n WHERE deleted_at IS NULL\n AND (\n owner_user_id = $1\n OR team_id IN (SELECT team_id FROM team_user WHERE user_id = $1)\n )\n ORDER BY created_at DESC\n ","describe": {"columns": [{@@ -57,6 +57,16 @@"ordinal": 10,"name": "last_disconnected_at","type_info": "Timestamptz"+ },+ {+ "ordinal": 11,+ "name": "has_live_connection!",+ "type_info": "Bool"+ },+ {+ "ordinal": 12,+ "name": "has_connection_record!",+ "type_info": "Bool"}],"parameters": {@@ -75,8 +85,10 @@false,false,true,- true+ true,+ null,+ null]},- "hash": "d30ffc9d1a8aabb56674f4a0292176684576a144a1dd05b4e6fed48f4e49e3a9"+ "hash": "1c08845abe8781e7beb18b8dbf5e367cf3b95acc7a26029a553efd8e2ca2b7a9"}diff --git a/.sqlx/query-65324cc6bff4a47702524a0ffc2b2e12acbd283bb0e90732e64cb74ad430ebcb.json b/.sqlx/query-65324cc6bff4a47702524a0ffc2b2e12acbd283bb0e90732e64cb74ad430ebcb.jsonnew file mode 100644index 0000000000..a8c618ecec--- /dev/null+++ b/.sqlx/query-65324cc6bff4a47702524a0ffc2b2e12acbd283bb0e90732e64cb74ad430ebcb.json@@ -0,0 +1,29 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH removed AS (\n DELETE FROM harness_runtime_connection\n WHERE harness_id = $1 AND (id = $2 OR lease_expires_at <= now())\n RETURNING id, connected_at\n ), others AS (\n SELECT count(*) AS live\n FROM harness_runtime_connection\n WHERE harness_id = $1 AND id <> $2 AND lease_expires_at > now()\n ), stamped AS (\n UPDATE harnesses SET last_disconnected_at = now()\n WHERE id = $1 AND (SELECT live FROM others) = 0\n )\n SELECT\n (SELECT live FROM others) AS \"other_live_connections!\",\n (\n SELECT (extract(epoch FROM now() - connected_at) * 1000)::bigint\n FROM removed\n WHERE id = $2\n ) AS connected_for_ms\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "other_live_connections!",+ "type_info": "Int8"+ },+ {+ "ordinal": 1,+ "name": "connected_for_ms",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid"+ ]+ },+ "nullable": [+ null,+ null+ ]+ },+ "hash": "65324cc6bff4a47702524a0ffc2b2e12acbd283bb0e90732e64cb74ad430ebcb"+}diff --git a/.sqlx/query-65ad06326d16861dbcce0f6d6c0a6e3459cb21197ed70222a139bbfc727e9c0d.json b/.sqlx/query-65ad06326d16861dbcce0f6d6c0a6e3459cb21197ed70222a139bbfc727e9c0d.jsonnew file mode 100644index 0000000000..109c658fa3--- /dev/null+++ b/.sqlx/query-65ad06326d16861dbcce0f6d6c0a6e3459cb21197ed70222a139bbfc727e9c0d.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT id FROM harnesses WHERE id = $1 FOR NO KEY UPDATE",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "65ad06326d16861dbcce0f6d6c0a6e3459cb21197ed70222a139bbfc727e9c0d"+}diff --git a/.sqlx/query-19c6aed4e629ba1a98572498d7803e1027d17a7215beb36d0e3b3c54cb635396.json b/.sqlx/query-88f86058dc56cda26b0c2be64b6b2e183027fc2dae082a901caa6c40c1d89d93.jsonsimilarity index 70%rename from .sqlx/query-19c6aed4e629ba1a98572498d7803e1027d17a7215beb36d0e3b3c54cb635396.jsonrename to .sqlx/query-88f86058dc56cda26b0c2be64b6b2e183027fc2dae082a901caa6c40c1d89d93.jsonindex 43158bb9c7..b69af0ee32 100644--- a/.sqlx/query-19c6aed4e629ba1a98572498d7803e1027d17a7215beb36d0e3b3c54cb635396.json+++ b/.sqlx/query-88f86058dc56cda26b0c2be64b6b2e183027fc2dae082a901caa6c40c1d89d93.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n UPDATE harnesses\n SET\n name = COALESCE($2, name),\n allow_permission_bypass = COALESCE($3, allow_permission_bypass),\n updated_at = now()\n WHERE id = $1 AND deleted_at IS NULL\n RETURNING\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at\n ",+ "query": "\n UPDATE harnesses\n SET\n name = COALESCE($2, name),\n allow_permission_bypass = COALESCE($3, allow_permission_bypass),\n updated_at = now()\n WHERE id = $1 AND deleted_at IS NULL\n RETURNING\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at,\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()\n ) AS \"has_live_connection!\",\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id\n ) AS \"has_connection_record!\"\n ","describe": {"columns": [{@@ -57,6 +57,16 @@"ordinal": 10,"name": "last_disconnected_at","type_info": "Timestamptz"+ },+ {+ "ordinal": 11,+ "name": "has_live_connection!",+ "type_info": "Bool"+ },+ {+ "ordinal": 12,+ "name": "has_connection_record!",+ "type_info": "Bool"}],"parameters": {@@ -77,8 +87,10 @@false,false,true,- true+ true,+ null,+ null]},- "hash": "19c6aed4e629ba1a98572498d7803e1027d17a7215beb36d0e3b3c54cb635396"+ "hash": "88f86058dc56cda26b0c2be64b6b2e183027fc2dae082a901caa6c40c1d89d93"}diff --git a/.sqlx/query-b4d134196b124c4c4a6b0ca5ceb6c8b0c14fbe1d11ad9c260c9108a27f64134d.json b/.sqlx/query-b4d134196b124c4c4a6b0ca5ceb6c8b0c14fbe1d11ad9c260c9108a27f64134d.jsondeleted file mode 100644index 7ce5d5b8f8..0000000000--- a/.sqlx/query-b4d134196b124c4c4a6b0ca5ceb6c8b0c14fbe1d11ad9c260c9108a27f64134d.json+++ /dev/null@@ -1,14 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "UPDATE harnesses SET last_disconnected_at = now() WHERE id = $1",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Uuid"- ]- },- "nullable": []- },- "hash": "b4d134196b124c4c4a6b0ca5ceb6c8b0c14fbe1d11ad9c260c9108a27f64134d"-}diff --git a/.sqlx/query-9c0a40ee4c671cdcd35dde8f6ef8278e3fc0a72e3f3ac82759719d660b4e4e5d.json b/.sqlx/query-c5052e4731eb9c665326a8884c1776feefaf57d261a5fe18c4a885feb4ffeed0.jsonsimilarity index 65%rename from .sqlx/query-9c0a40ee4c671cdcd35dde8f6ef8278e3fc0a72e3f3ac82759719d660b4e4e5d.jsonrename to .sqlx/query-c5052e4731eb9c665326a8884c1776feefaf57d261a5fe18c4a885feb4ffeed0.jsonindex 26b3b367ae..0716c53614 100644--- a/.sqlx/query-9c0a40ee4c671cdcd35dde8f6ef8278e3fc0a72e3f3ac82759719d660b4e4e5d.json+++ b/.sqlx/query-c5052e4731eb9c665326a8884c1776feefaf57d261a5fe18c4a885feb4ffeed0.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n SELECT\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at\n FROM harnesses\n WHERE id = $1 AND deleted_at IS NULL\n ",+ "query": "\n SELECT\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at,\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()\n ) AS \"has_live_connection!\",\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id\n ) AS \"has_connection_record!\"\n FROM harnesses\n WHERE id = $1 AND deleted_at IS NULL\n ","describe": {"columns": [{@@ -57,6 +57,16 @@"ordinal": 10,"name": "last_disconnected_at","type_info": "Timestamptz"+ },+ {+ "ordinal": 11,+ "name": "has_live_connection!",+ "type_info": "Bool"+ },+ {+ "ordinal": 12,+ "name": "has_connection_record!",+ "type_info": "Bool"}],"parameters": {@@ -75,8 +85,10 @@false,false,true,- true+ true,+ null,+ null]},- "hash": "9c0a40ee4c671cdcd35dde8f6ef8278e3fc0a72e3f3ac82759719d660b4e4e5d"+ "hash": "c5052e4731eb9c665326a8884c1776feefaf57d261a5fe18c4a885feb4ffeed0"}diff --git a/.sqlx/query-de149836f561243b3afb7c1b1b8f74e79f1a89a7097a81754ceb38ba85008990.json b/.sqlx/query-de149836f561243b3afb7c1b1b8f74e79f1a89a7097a81754ceb38ba85008990.jsonnew file mode 100644index 0000000000..539f6fe274--- /dev/null+++ b/.sqlx/query-de149836f561243b3afb7c1b1b8f74e79f1a89a7097a81754ceb38ba85008990.json@@ -0,0 +1,17 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH reaped AS (\n DELETE FROM harness_runtime_connection\n WHERE harness_id = $1 AND lease_expires_at <= now()\n ), added AS (\n INSERT INTO harness_runtime_connection\n (harness_id, id, replica_id, lease_expires_at)\n VALUES ($1, $2, $3, now() + make_interval(secs => $4))\n ON CONFLICT (harness_id, id) DO NOTHING\n )\n UPDATE harnesses SET last_connected_at = now() WHERE id = $1\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid",+ "Uuid",+ "Float8"+ ]+ },+ "nullable": []+ },+ "hash": "de149836f561243b3afb7c1b1b8f74e79f1a89a7097a81754ceb38ba85008990"+}diff --git a/.sqlx/query-e40e811009a56836ce10a1cd8d731a74aaaf7f7ddda8ff82abb89d6c10fc8524.json b/.sqlx/query-e40e811009a56836ce10a1cd8d731a74aaaf7f7ddda8ff82abb89d6c10fc8524.jsonnew file mode 100644index 0000000000..1dc9029ec2--- /dev/null+++ b/.sqlx/query-e40e811009a56836ce10a1cd8d731a74aaaf7f7ddda8ff82abb89d6c10fc8524.json@@ -0,0 +1,37 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO harness_runtime_connection\n (harness_id, id, replica_id, lease_expires_at)\n SELECT live.harness_id, live.id, $3, now() + make_interval(secs => $4)\n FROM UNNEST($1::uuid[], $2::uuid[]) AS live(harness_id, id)\n JOIN harnesses ON harnesses.id = live.harness_id\n ON CONFLICT (harness_id, id) DO UPDATE\n SET lease_expires_at = EXCLUDED.lease_expires_at\n RETURNING harness_id, id, (xmax = 0) AS \"restored!\"\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "harness_id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 2,+ "name": "restored!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "UuidArray",+ "UuidArray",+ "Uuid",+ "Float8"+ ]+ },+ "nullable": [+ false,+ false,+ null+ ]+ },+ "hash": "e40e811009a56836ce10a1cd8d731a74aaaf7f7ddda8ff82abb89d6c10fc8524"+}diff --git a/.sqlx/query-f1bfc93ff556e8fc22dbc7788dc5474aa042c379d0f255faa0b487c5480bffcb.json b/.sqlx/query-f1bfc93ff556e8fc22dbc7788dc5474aa042c379d0f255faa0b487c5480bffcb.jsondeleted file mode 100644index 9512791b63..0000000000--- a/.sqlx/query-f1bfc93ff556e8fc22dbc7788dc5474aa042c379d0f255faa0b487c5480bffcb.json+++ /dev/null@@ -1,14 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "UPDATE harnesses SET last_connected_at = now() WHERE id = $1",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Uuid"- ]- },- "nullable": []- },- "hash": "f1bfc93ff556e8fc22dbc7788dc5474aa042c379d0f255faa0b487c5480bffcb"-}diff --git a/crates/agent_harness/AGENTS.md b/crates/agent_harness/AGENTS.mdindex cb16e7000d..3dd0ef8e60 100644--- a/crates/agent_harness/AGENTS.md+++ b/crates/agent_harness/AGENTS.md@@ -94,6 +94,16 @@ Reconcile active leases against terminal log evidence too: an observer can misscompletion during takeover. Previously bound quiet attempts reconcile before newwakes; interrupted quiet batches return to the next real prompt.+Harness presence is per connection, because a runtime's redial can land on a+different replica than the socket it replaced, and a stalled replica can notice+that socket's end long after. `RuntimeRegistry` mints a `RuntimeConnectionId` per+dial and reports it to `HarnessPresence`; a close removes only its own row, and+each replica renews the leases of what it holds, so a crash reads offline within+the lease. To debug a harness in Datadog, search `@harness:<id>` for+`@telemetry_event:(harness_runtime_connected OR harness_runtime_disconnected OR+harness_runtime_lease_restored)`: each carries `connection` and `replica`, and a+disconnect's `other_live_connections` above 0 is a stale close that changed nothing.+## Checks- `cargo nextest run -p agent_harness --lib --test-threads=2` (`SQLX_OFFLINE`diff --git a/crates/agent_harness/src/domain/ports.rs b/crates/agent_harness/src/domain/ports.rsindex 48cdad2230..6145e06d37 100644--- a/crates/agent_harness/src/domain/ports.rs+++ b/crates/agent_harness/src/domain/ports.rs@@ -143,20 +143,58 @@ pub trait CodingAgentSource: Send + Sync + 'static {) -> impl Future<Output = anyhow::Result<Option<bool>>> + Send;}+/// One dial of a harness's runtime; a redial is a new connection.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct RuntimeConnectionId(macro_uuid::Uuid);++impl RuntimeConnectionId {+ /// Mint the identity of a new dial.+ #[must_use]+ pub fn generate() -> Self {+ Self(macro_uuid::generate_uuid_v7())+ }++ /// The raw uuid, for persistence.+ #[must_use]+ pub fn as_uuid(self) -> macro_uuid::Uuid {+ self.0+ }+}++impl std::fmt::Display for RuntimeConnectionId {+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ self.0.fmt(formatter)+ }+}+/// Durable attach/detach bookkeeping for harness runtime connections.////// The registry itself is in-process liveness; this is what lets the rest of-/// the product (the harness settings page) see whether a daemon is up.+/// the product (the harness settings page) see whether a daemon is up. Each+/// report names its connection because a harness's redial can land on another+/// replica, which then learns nothing about the older socket being closed here./// Methods take `Arc<Self>` and return owned futures so the registry can fire/// them from its own background tasks.pub trait HarnessPresence: Send + Sync + 'static {/// A runtime attached for this harness.- fn connected(self: Arc<Self>, harness: HarnessId) -> Pin<Box<dyn Future<Output = ()> + Send>>;+ fn connected(+ self: Arc<Self>,+ harness: HarnessId,+ connection: RuntimeConnectionId,+ ) -> Pin<Box<dyn Future<Output = ()> + Send>>;- /// This harness's runtime connection closed.+ /// This connection closed. Other connections of the harness are unaffected.fn disconnected(self: Arc<Self>,harness: HarnessId,+ connection: RuntimeConnectionId,+ ) -> Pin<Box<dyn Future<Output = ()> + Send>>;++ /// These connections are still held by this process, so a crash of this+ /// process stops vouching for them.+ fn renew(+ self: Arc<Self>,+ live: Vec<(HarnessId, RuntimeConnectionId)>,) -> Pin<Box<dyn Future<Output = ()> + Send>>;}diff --git a/crates/agent_harness/src/outbound/runtime_registry.rs b/crates/agent_harness/src/outbound/runtime_registry.rsindex 9025abb4d5..527e021049 100644--- a/crates/agent_harness/src/outbound/runtime_registry.rs+++ b/crates/agent_harness/src/outbound/runtime_registry.rs@@ -22,8 +22,11 @@ use bot_id::BotId;use dashmap::DashMap;use harness_id::HarnessId;use tokio::sync::mpsc;+use tokio::task::JoinHandle;-use crate::domain::ports::{HarnessBindings, HarnessPresence, RuntimeConnections};+use crate::domain::ports::{+ HarnessBindings, HarnessPresence, RuntimeConnectionId, RuntimeConnections,+};#[cfg(test)]mod test;@@ -42,9 +45,25 @@ pub struct LiveRuntimeSessions {pub span: tracing::Span,}+/// A connection and the identity its presence reports carry.+struct ListedConnection<Sender> {+ id: RuntimeConnectionId,+ connection: Arc<RuntimeConnection<Sender>>,+}++// Manual Clone impl: the sender itself is behind the Arc.+impl<Sender> Clone for ListedConnection<Sender> {+ fn clone(&self) -> Self {+ Self {+ id: self.id,+ connection: Arc::clone(&self.connection),+ }+ }+}+/// Every harness's live runtime connection.pub struct RuntimeRegistry<Sender> {- connections: DashMap<HarnessId, Arc<RuntimeConnection<Sender>>>,+ connections: DashMap<HarnessId, ListedConnection<Sender>>,presence: Option<Arc<dyn HarnessPresence>>,live_sessions: Option<mpsc::UnboundedSender<LiveRuntimeSessions>>,}@@ -117,7 +136,7 @@ wherelet connection = self.connections.get(&harness)- .map(|entry| Arc::clone(&entry))?;+ .map(|entry| Arc::clone(&entry.connection))?;Some(connection.probe_models().await)}}@@ -136,26 +155,51 @@ wherewhereCarrier: Transport<ToRuntimeMessage, ToServerMessage, Sender = Sender>,{- let connection = RuntimeConnection::connect(carrier);+ let listed = ListedConnection {+ id: RuntimeConnectionId::generate(),+ connection: RuntimeConnection::connect(carrier),+ };// Listed before it is watched: a socket that dies immediately would// otherwise have its eviction run before the entry it is meant to// remove exists, leaving the dead connection listed for good.- if let Some(displaced) = self.connections.insert(harness, Arc::clone(&connection)) {- displaced.evict();- tracing::info!(%harness, "a redialed runtime replaced this harness's connection");- }- if let Some(presence) = &self.presence {- tokio::spawn(Arc::clone(presence).connected(harness));+ if let Some(displaced) = self.connections.insert(harness, listed.clone()) {+ displaced.connection.evict();+ tracing::info!(+ %harness,+ connection = %listed.id,+ displaced_connection = %displaced.id,+ "a redialed runtime replaced this harness's connection"+ );}+ let connected_write = self+ .presence+ .as_ref()+ .map(|presence| tokio::spawn(Arc::clone(presence).connected(harness, listed.id)));if let Some(live_sessions) = &self.live_sessions {tokio::spawn(report_live_sessions(harness,- Arc::clone(&connection),+ Arc::clone(&listed.connection),live_sessions.clone(),));}- self.watch_for_close(harness, connection);+ self.watch_for_close(harness, listed, connected_write);+ }++ /// Tell presence which connections this process still holds.+ pub async fn renew_presence(&self) {+ let Some(presence) = &self.presence else {+ return;+ };++ let live: Vec<_> = self+ .connections+ .iter()+ .map(|entry| (*entry.key(), entry.id))+ .collect();+ if !live.is_empty() {+ Arc::clone(presence).renew(live).await;+ }}/// Drop `connection` from the registry once its transport ends.@@ -168,9 +212,10 @@ wherefn watch_for_close(self: &Arc<Self>,harness: HarnessId,- connection: Arc<RuntimeConnection<Sender>>,+ listed: ListedConnection<Sender>,+ connected_write: Option<JoinHandle<()>>,) {- tokio::spawn(Arc::clone(self).drop_when_closed(harness, connection));+ tokio::spawn(Arc::clone(self).drop_when_closed(harness, listed, connected_write));}/// The body of that watch: wait for the end, then unlist it.@@ -180,23 +225,31 @@ whereasync fn drop_when_closed(self: Arc<Self>,harness: HarnessId,- connection: Arc<RuntimeConnection<Sender>>,+ listed: ListedConnection<Sender>,+ connected_write: Option<JoinHandle<()>>,) {- connection.closed().await;+ listed.connection.closed().await;// Only if it is still this connection: a redial that already replaced// it owns the entry now, and evicting that would take the live socket- // down along with the dead one. The presence write follows the same- // rule, so a displaced socket's death never marks the live redial- // disconnected.+ // down along with the dead one.let removed = self.connections- .remove_if(&harness, |_, current| Arc::ptr_eq(current, &connection))+ .remove_if(&harness, |_, current| current.id == listed.id).is_some();- if removed {- if let Some(presence) = &self.presence {- tokio::spawn(Arc::clone(presence).disconnected(harness));+ if let Some(presence) = &self.presence {+ // A socket that dies immediately must not have its close recorded+ // before its attach, which would leave the attach standing.+ if let Some(connected_write) = connected_write {+ let _ = connected_write.await;}- tracing::info!(%harness, "dropped this harness's closed runtime connection");+ Arc::clone(presence).disconnected(harness, listed.id).await;+ }+ if removed {+ tracing::info!(+ %harness,+ connection = %listed.id,+ "dropped this harness's closed runtime connection"+ );}}}@@ -281,7 +334,7 @@ where.registry.connections.get(&harness)- .map(|entry| Arc::clone(&entry))?;+ .map(|entry| Arc::clone(&entry.connection))?;Some(connection.bind(session).await)}diff --git a/crates/agent_harness/src/outbound/runtime_registry/test.rs b/crates/agent_harness/src/outbound/runtime_registry/test.rsindex 27978644f9..f5df5702c2 100644--- a/crates/agent_harness/src/outbound/runtime_registry/test.rs+++ b/crates/agent_harness/src/outbound/runtime_registry/test.rs@@ -164,7 +164,7 @@ async fn a_closed_connection_is_unlisted_so_nothing_binds_onto_it() {let connection = registry.connections.get(&HarnessId::TEST_A)- .map(|entry| Arc::clone(&entry))+ .map(|entry| entry.value().clone()).expect("the dial is listed");let bound = connections(Arc::new(FakeBindings::bound(BotId::TEST_A, HarnessId::TEST_A)),@@ -174,7 +174,7 @@ async fn a_closed_connection_is_unlisted_so_nothing_binds_onto_it() {runtime.disconnects();// The same work `attach` spawns, awaited here rather than raced.Arc::clone(®istry)- .drop_when_closed(HarnessId::TEST_A, connection)+ .drop_when_closed(HarnessId::TEST_A, connection, None).await;assert!(!registry.is_connected(HarnessId::TEST_A));@@ -194,7 +194,7 @@ async fn a_redials_connection_outlives_the_one_it_displaced() {let displaced = registry.connections.get(&HarnessId::TEST_A)- .map(|entry| Arc::clone(&entry))+ .map(|entry| entry.value().clone()).expect("the first dial is listed");let bound = connections(Arc::new(FakeBindings::bound(BotId::TEST_A, HarnessId::TEST_A)),@@ -204,7 +204,7 @@ async fn a_redials_connection_outlives_the_one_it_displaced() {registry.attach(HarnessId::TEST_A, ContainerMock::default());first.disconnects();Arc::clone(®istry)- .drop_when_closed(HarnessId::TEST_A, displaced)+ .drop_when_closed(HarnessId::TEST_A, displaced, None).await;// The dead connection's eviction must not unlist the live one that took@@ -218,61 +218,167 @@ async fn a_redials_connection_outlives_the_one_it_displaced() {);}-#[tokio::test]-async fn attach_and_close_report_presence() {- #[derive(Default)]- struct RecordingPresence {- events: Mutex<Vec<(HarnessId, &'static str)>>,+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum PresenceEvent {+ Connected(RuntimeConnectionId),+ Disconnected(RuntimeConnectionId),+}++#[derive(Default)]+struct RecordingPresence {+ events: Mutex<Vec<PresenceEvent>>,+ renewed: Mutex<Vec<Vec<(HarnessId, RuntimeConnectionId)>>>,+}++impl HarnessPresence for RecordingPresence {+ fn connected(+ self: Arc<Self>,+ _harness: HarnessId,+ connection: RuntimeConnectionId,+ ) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>> {+ Box::pin(async move {+ self.events+ .lock()+ .unwrap()+ .push(PresenceEvent::Connected(connection));+ })}- impl HarnessPresence for RecordingPresence {- fn connected(- self: Arc<Self>,- harness: HarnessId,- ) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>> {- Box::pin(async move {- self.events.lock().unwrap().push((harness, "connected"));- })- }+ fn disconnected(+ self: Arc<Self>,+ _harness: HarnessId,+ connection: RuntimeConnectionId,+ ) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>> {+ Box::pin(async move {+ self.events+ .lock()+ .unwrap()+ .push(PresenceEvent::Disconnected(connection));+ })+ }- fn disconnected(- self: Arc<Self>,- harness: HarnessId,- ) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>> {- Box::pin(async move {- self.events.lock().unwrap().push((harness, "disconnected"));- })- }+ fn renew(+ self: Arc<Self>,+ live: Vec<(HarnessId, RuntimeConnectionId)>,+ ) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>> {+ Box::pin(async move {+ self.renewed.lock().unwrap().push(live);+ })}+}- let presence = Arc::new(RecordingPresence::default());- let registry: Arc<RuntimeRegistry<ContainerSender>> = RuntimeRegistry::with_presence(- Arc::clone(&presence) as Arc<dyn HarnessPresence>,+fn registry_with_presence(+ presence: &Arc<RecordingPresence>,+) -> Arc<RuntimeRegistry<ContainerSender>> {+ RuntimeRegistry::with_presence(+ Arc::clone(presence) as Arc<dyn HarnessPresence>,mpsc::unbounded_channel().0,- );- let runtime = ContainerMock::default();- registry.attach(HarnessId::TEST_A, runtime.clone());- let connection = registry+ )+}++fn listed_connection(+ registry: &Arc<RuntimeRegistry<ContainerSender>>,+) -> ListedConnection<ContainerSender> {+ registry.connections.get(&HarnessId::TEST_A)- .map(|entry| Arc::clone(&entry))- .expect("the dial is listed");+ .map(|entry| entry.value().clone())+ .expect("the dial is listed")+}- runtime.disconnects();+#[tokio::test]+async fn a_displaced_connections_close_reports_only_that_connection() {+ let presence = Arc::new(RecordingPresence::default());+ let registry = registry_with_presence(&presence);+ let first = ContainerMock::default();+ registry.attach(HarnessId::TEST_A, first.clone());+ let displaced = listed_connection(®istry);+ registry.attach(HarnessId::TEST_A, ContainerMock::default());+ let current = listed_connection(®istry);++ first.disconnects();Arc::clone(®istry)- .drop_when_closed(HarnessId::TEST_A, connection)+ .drop_when_closed(HarnessId::TEST_A, displaced.clone(), None).await;- // The connected write is spawned; give it a beat to land.- for _ in 0..100 {- if presence.events.lock().unwrap().len() >= 2 {- break;+ // Presence is told about the dead connection, by name, so its write cannot+ // be mistaken for the live connection going away.+ let events = presence.events.lock().unwrap().clone();+ assert!(events.contains(&PresenceEvent::Disconnected(displaced.id)));+ assert!(!events.contains(&PresenceEvent::Disconnected(current.id)));+ assert_eq!(listed_connection(®istry).id, current.id);+}++#[tokio::test(start_paused = true)]+async fn a_close_waits_for_its_own_connected_write() {+ let presence = Arc::new(RecordingPresence::default());+ let registry = registry_with_presence(&presence);+ let runtime = ContainerMock::default();+ // Not through `attach`, whose own watch would report the close unprompted.+ let listed = ListedConnection {+ id: RuntimeConnectionId::generate(),+ connection: RuntimeConnection::connect(runtime.clone()),+ };+ registry+ .connections+ .insert(HarnessId::TEST_A, listed.clone());+ let (release, released) = tokio::sync::oneshot::channel::<()>();+ let connected_write = tokio::spawn({+ let presence = Arc::clone(&presence);+ let id = listed.id;+ async move {+ let _ = released.await;+ presence+ .events+ .lock()+ .unwrap()+ .push(PresenceEvent::Connected(id));}- tokio::task::yield_now().await;+ });+ runtime.disconnects();++ let closing = Arc::clone(®istry).drop_when_closed(+ HarnessId::TEST_A,+ listed.clone(),+ Some(connected_write),+ );+ tokio::pin!(closing);+ tokio::select! {+ () = &mut closing => panic!("the close finished before its attach was recorded"),+ () = tokio::time::sleep(Duration::from_secs(1)) => {}}- let events = presence.events.lock().unwrap().clone();- assert!(events.contains(&(HarnessId::TEST_A, "connected")));- assert!(events.contains(&(HarnessId::TEST_A, "disconnected")));+ // The registry already stopped listing it; only the presence write waited.+ assert!(!registry.is_connected(HarnessId::TEST_A));+ assert!(presence.events.lock().unwrap().is_empty());++ release.send(()).expect("the connected write is waiting");+ closing.await;++ assert_eq!(+ *presence.events.lock().unwrap(),+ [+ PresenceEvent::Connected(listed.id),+ PresenceEvent::Disconnected(listed.id)+ ]+ );+}++#[tokio::test]+async fn renewal_lists_the_connections_this_process_holds() {+ let presence = Arc::new(RecordingPresence::default());+ let registry = registry_with_presence(&presence);++ registry.renew_presence().await;+ assert!(presence.renewed.lock().unwrap().is_empty());++ registry.attach(HarnessId::TEST_A, ContainerMock::default());+ let listed = listed_connection(®istry);+ registry.renew_presence().await;++ assert_eq!(+ *presence.renewed.lock().unwrap(),+ [vec![(HarnessId::TEST_A, listed.id)]]+ );}#[tokio::test]diff --git a/crates/coding_agent_worker/AGENTS.md b/crates/coding_agent_worker/AGENTS.mdindex ec8526a1b4..9302df8910 100644--- a/crates/coding_agent_worker/AGENTS.md+++ b/crates/coding_agent_worker/AGENTS.md@@ -26,6 +26,11 @@ the local ACP process (Claude Code, Codex, ...), and drives nativewhat the session has. General Macro guidance stays in the `SelfKnowledge` tool.- `src/trigger.rs` + `src/dispatch.rs`: agent-trigger broker events → sessioncreate/dial/prompt against the live harness service.+- `src/daemon.rs` `watch_presence`: each minute a connection has held, asks+ `GET /harnesses/me` with `RUNTIME_CONNECTED_HEADER`. A service that reads the+ harness offline logs `harness_presence_disagreement`, macrod warns, and the TUI+ header says "runtime connected, service reports offline" instead of "paired,+ runtime not dialed in".- `src/tui.rs` (`src/tui/`): the control panel itself (registration, agents,sessions, config form, logs).- `src/herdr.rs` (`src/herdr/`): native herdr integration — only compileddiff --git a/crates/coding_agent_worker/src/daemon.rs b/crates/coding_agent_worker/src/daemon.rsindex acbf9c2c9c..460f0ad741 100644--- a/crates/coding_agent_worker/src/daemon.rs+++ b/crates/coding_agent_worker/src/daemon.rs@@ -7,7 +7,9 @@use std::path::Path;use std::time::Duration;+use tokio::sync::watch;use tokio_util::sync::CancellationToken;+use tokio_util::task::AbortOnDropHandle;use crate::config::{Config, HarnessCredentials};use crate::dispatch::Dispatcher;@@ -28,6 +30,9 @@ mod test;/// worst-case window where a brand-new agent is deaf.const BOUND_AGENT_REFRESH: Duration = Duration::from_secs(10);+/// How often a held runtime connection is compared with what the service says.+const PRESENCE_CHECK_EVERY: Duration = Duration::from_secs(60);+/// A running serving core: SSE listener, harness bridge.pub struct Daemon {cancel: CancellationToken,@@ -38,6 +43,8 @@ pub struct Daemon {expect(dead_code, reason = "agents are handed off only on Unix"))]handoff: HandoffSender,+ runtime_connected: watch::Receiver<bool>,+ _presence_watch: AbortOnDropHandle<()>,}/// The agent a previous macrod process image handed over, ready to relay.@@ -102,6 +109,11 @@ impl Daemon {adopted,handoffs,);+ let runtime_connected = runtime.connection_state();+ let presence_watch = AbortOnDropHandle::new(tokio::spawn(watch_presence(+ EventStreamClient::new(&config.macro_api, &credentials),+ runtime_connected.clone(),+ )));let executor = Dispatcher::new(api, runtime, config.workspace.clone());#[cfg(unix)]let executor = executor.with_workspaces(workspaces);@@ -150,9 +162,16 @@ impl Daemon {task,turns,handoff,+ runtime_connected,+ _presence_watch: presence_watch,})}+ /// Whether this daemon currently holds a runtime connection.+ pub fn runtime_connected(&self) -> bool {+ *self.runtime_connected.borrow()+ }+/// Whether the serving task is still alive.pub fn is_running(&self) -> bool {!self.task.is_finished()@@ -195,6 +214,47 @@ impl Daemon {}}+/// Compare a held runtime connection with what the service reports.+///+/// The service reading this harness as offline makes the app treat its agents as+/// unavailable, and nothing else on this machine would notice: the socket is+/// healthy. Warns once per disagreement; the check itself lets the service log+/// each one for its monitor.+async fn watch_presence(client: EventStreamClient, mut runtime: watch::Receiver<bool>) {+ let mut ticker = tokio::time::interval(PRESENCE_CHECK_EVERY);+ ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);+ let mut disagreeing = false;+ loop {+ ticker.tick().await;++ if !runtime_held_since_last_look(&mut runtime) {+ disagreeing = false;+ continue;+ }++ match client.service_reports_connected().await {+ Ok(false) if !disagreeing => {+ disagreeing = true;+ tracing::warn!("macrod holds a runtime connection the service reports as offline");+ }+ Ok(true) if disagreeing => {+ disagreeing = false;+ tracing::info!("the service reports this harness connected again");+ }+ Ok(_) => {}+ Err(error) => tracing::debug!(error = %error, "could not check harness presence"),+ }+ }+}++/// Whether the runtime has stayed connected since this was last asked. A fresh+/// dial does not count yet: the service's record of it may still be landing.+fn runtime_held_since_last_look(runtime: &mut watch::Receiver<bool>) -> bool {+ let redialed = runtime.has_changed().unwrap_or(true);+ let connected = *runtime.borrow_and_update();+ connected && !redialed+}+/// Relay an inherited agent instead of spawning one, continuing its turns.#[cfg(unix)]fn adopt(inherited: Option<InheritedAgent>, turns: &ActiveTurns) -> Option<RunningAgent> {diff --git a/crates/coding_agent_worker/src/daemon/test.rs b/crates/coding_agent_worker/src/daemon/test.rsindex 2da8815a4d..ca47cb97be 100644--- a/crates/coding_agent_worker/src/daemon/test.rs+++ b/crates/coding_agent_worker/src/daemon/test.rs@@ -1,5 +1,24 @@use super::*;+#[test]+fn a_fresh_dial_is_not_a_held_connection_until_it_has_lasted() {+ let (connected, mut runtime) = watch::channel(false);+ assert!(!runtime_held_since_last_look(&mut runtime));++ connected.send_replace(true);+ assert!(!runtime_held_since_last_look(&mut runtime));+ assert!(runtime_held_since_last_look(&mut runtime));++ // A drop and redial between looks starts the wait over.+ connected.send_replace(false);+ connected.send_replace(true);+ assert!(!runtime_held_since_last_look(&mut runtime));+ assert!(runtime_held_since_last_look(&mut runtime));++ connected.send_replace(false);+ assert!(!runtime_held_since_last_look(&mut runtime));+}+#[tokio::test]async fn stop_does_not_wait_for_in_flight_work() {let daemon = Daemon {@@ -7,6 +26,8 @@ async fn stop_does_not_wait_for_in_flight_work() {task: tokio::spawn(std::future::pending()),turns: ActiveTurns::default(),handoff: crate::harness::Handoffs::channel().1,+ runtime_connected: watch::channel(false).1,+ _presence_watch: AbortOnDropHandle::new(tokio::spawn(async {})),};tokio::time::timeout(Duration::from_secs(1), daemon.stop())diff --git a/crates/coding_agent_worker/src/outbound/stream.rs b/crates/coding_agent_worker/src/outbound/stream.rsindex f3027a8bc2..7e8ff469e6 100644--- a/crates/coding_agent_worker/src/outbound/stream.rs+++ b/crates/coding_agent_worker/src/outbound/stream.rs@@ -10,10 +10,11 @@use std::collections::BTreeSet;use std::marker::PhantomData;use std::pin::Pin;+use std::time::Duration;use eventsource_stream::{Event, EventStreamError, Eventsource as _};use futures::{Stream, StreamExt as _};-use harnesses::domain::models::{Harness, HarnessAgent, HarnessOwner};+use harnesses::domain::models::{Harness, HarnessAgent, HarnessOwner, RUNTIME_CONNECTED_HEADER};use rootcause::prelude::ResultExt as _;use serde::de::DeserializeOwned;use webhook::domain::models::{WebhookFilter, WebhookScope};@@ -25,6 +26,10 @@ mod test;const HARNESS_TOKEN_HEADER: &str = "x-macro-harness-token";+/// Set on the presence request alone: a timeout on the client would also end the+/// long-lived event stream.+const PRESENCE_CHECK_TIMEOUT: Duration = Duration::from_secs(10);+/// Map the harness's ownership onto the stream query's workspace scope.pub fn stream_scope(owner: &HarnessOwner) -> WebhookScope {match owner {@@ -94,6 +99,21 @@ impl EventStreamClient {Ok(stream_scope(&harness.owner))}+ /// Whether the service reports this harness connected, telling it that this+ /// daemon holds a runtime connection so it can log any disagreement.+ pub async fn service_reports_connected(&self) -> rootcause::Result<bool> {+ let harness: Harness = self+ .read(+ "check this harness's presence",+ self.http+ .get(format!("{}/harnesses/me", self.base))+ .header(RUNTIME_CONNECTED_HEADER, "true")+ .timeout(PRESENCE_CHECK_TIMEOUT),+ )+ .await?;+ Ok(harness.connected)+ }+/// The agents currently bound to this harness, as sorted bot-id strings —/// the shape the stream filter carries.pub async fn bound_bot_ids(&self) -> rootcause::Result<Vec<String>> {diff --git a/crates/coding_agent_worker/src/outbound/stream/test.rs b/crates/coding_agent_worker/src/outbound/stream/test.rsindex f35bff745e..01426c9a5f 100644--- a/crates/coding_agent_worker/src/outbound/stream/test.rs+++ b/crates/coding_agent_worker/src/outbound/stream/test.rs@@ -29,6 +29,33 @@ struct Envelope {n: u8,}+#[tokio::test]+async fn presence_check_tells_the_service_a_connection_is_held() {+ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};++ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind test listener");+ let url = format!(+ "http://{}",+ listener.local_addr().expect("listener address")+ );+ let server = tokio::spawn(async move {+ let (mut socket, _) = listener.accept().await.expect("accept");+ let mut buf = vec![0u8; 4096];+ let read = socket.read(&mut buf).await.unwrap_or(0);+ let _ = socket.write_all(TEAM_HARNESS.as_bytes()).await;+ let _ = socket.shutdown().await;+ String::from_utf8_lossy(&buf[..read]).to_lowercase()+ });++ assert!(test_client(&url).service_reports_connected().await.unwrap());++ let request = server.await.unwrap();+ assert!(request.starts_with("get /harnesses/me "));+ assert!(request.contains("x-macro-harness-runtime-connected: true"));+}+#[tokio::test]async fn connect_yields_typed_envelopes_and_skips_keep_alives_and_junk() {let (url, requests) = serve(vec![diff --git a/crates/coding_agent_worker/src/runtime.rs b/crates/coding_agent_worker/src/runtime.rsindex 596e01b6e3..c832c16b63 100644--- a/crates/coding_agent_worker/src/runtime.rs+++ b/crates/coding_agent_worker/src/runtime.rs@@ -66,6 +66,11 @@ impl Runtime {}}+ /// Follows whether the runtime connection is currently up.+ pub fn connection_state(&self) -> watch::Receiver<bool> {+ self.connected.clone()+ }+/// Wait briefly for the supervisor to connect before dispatching a prompt./// Callers never open a second connection while it is reconnecting.pub async fn ensure_connected(&self) -> Result<(), tungstenite::Error> {diff --git a/crates/coding_agent_worker/src/tui/api.rs b/crates/coding_agent_worker/src/tui/api.rsindex fc1a4fa555..f72bcc4797 100644--- a/crates/coding_agent_worker/src/tui/api.rs+++ b/crates/coding_agent_worker/src/tui/api.rs@@ -1,5 +1,7 @@//! The TUI's read/act client against the harness APIs, as the harness.+use std::time::Duration;+use harnesses::domain::models::{Harness, HarnessAgent, HarnessSession};use reqwest::StatusCode;use rootcause::prelude::ResultExt as _;@@ -9,6 +11,10 @@ use crate::config::HarnessCredentials;const HARNESS_TOKEN_HEADER: &str = "x-macro-harness-token";+/// A dashboard read that takes longer than this is not worth waiting on: the+/// next refresh asks again.+const READ_TIMEOUT: Duration = Duration::from_secs(10);+/// What the server currently knows about this daemon.#[derive(Debug, Clone, Default)]pub struct Snapshot {@@ -50,9 +56,15 @@ impl HarnessSelfApi {.http.get(format!("{}{path}", self.base)).header(HARNESS_TOKEN_HEADER, &self.token)+ .timeout(READ_TIMEOUT).send().await- .context(format!("could not reach the service to {what}"))?;+ .map_err(|error| {+ rootcause::report!(+ "could not reach the service to {what}: {}",+ transport_failure(error)+ )+ })?;// A dead credential (revoked harness) answers 401; surface that as// "no harness" rather than an error so the UI offers re-pairing.if response.status() == StatusCode::UNAUTHORIZED@@ -114,3 +126,18 @@ impl HarnessSelfApi {Ok(())}}++/// A transport error with its causes on one line. reqwest's own message stops at+/// "error sending request for url", leaving out the reason, and the URL it leads+/// with would push the reason out of the one-line footer that shows this.+fn transport_failure(error: reqwest::Error) -> String {+ let error = error.without_url();+ let mut failure = error.to_string();+ let mut source = std::error::Error::source(&error);+ while let Some(cause) = source {+ failure.push_str(": ");+ failure.push_str(&cause.to_string());+ source = cause.source();+ }+ failure+}diff --git a/crates/coding_agent_worker/src/tui/app.rs b/crates/coding_agent_worker/src/tui/app.rsindex 2001573b96..d288be0edf 100644--- a/crates/coding_agent_worker/src/tui/app.rs+++ b/crates/coding_agent_worker/src/tui/app.rs@@ -21,6 +21,7 @@ use crate::outbound::pairing::{ClaimStatus, PairingClient, machine_name};/// How often the dashboard re-reads the server.pub(crate) const REFRESH_EVERY: Duration = Duration::from_secs(3);+const REFRESH_FAILED: &str = "refresh failed";/// Which tab is showing.#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -174,6 +175,11 @@ impl App {}/// (Re)start the serving core on the current config and credential.+ /// Whether this process currently holds a runtime connection.+ pub(crate) fn runtime_connected(&self) -> bool {+ self.daemon.as_ref().is_some_and(Daemon::runtime_connected)+ }+pub(crate) async fn restart_daemon(&mut self) -> bool {if let Some(daemon) = self.daemon.take() {daemon.stop().await;@@ -250,11 +256,28 @@ impl App {Ok(snapshot) => {self.snapshot = snapshot;self.last_refresh = Some(Instant::now());+ // A failure this refresh recovered from is no longer news, and+ // left up it reads as a live problem.+ if self.showing_refresh_failure() {+ self.status = None;+ }+ }+ Err(error) => {+ // Failed refreshes retry on every tick; log the first of a run.+ if !self.showing_refresh_failure() {+ tracing::warn!(error = %error, "could not refresh the dashboard");+ }+ self.fail(format!("{REFRESH_FAILED}: {error}"));}- Err(error) => self.fail(format!("refresh failed: {error}")),}}+ fn showing_refresh_failure(&self) -> bool {+ self.status+ .as_ref()+ .is_some_and(|(message, _)| message.starts_with(REFRESH_FAILED))+ }+pub(crate) async fn maybe_refresh(&mut self) {let due = self.last_refreshdiff --git a/crates/coding_agent_worker/src/tui/ui/shell.rs b/crates/coding_agent_worker/src/tui/ui/shell.rsindex 0288f3676c..3d9e49c586 100644--- a/crates/coding_agent_worker/src/tui/ui/shell.rs+++ b/crates/coding_agent_worker/src/tui/ui/shell.rs@@ -10,6 +10,11 @@ use crate::tui::app::{App, Mode, Tab};pub(super) fn render_header(frame: &mut Frame, app: &App, area: Rect) {let (dot, dot_style, state) = match &app.snapshot.harness {Some(harness) if harness.connected => ("●", Style::new().fg(OK), "connected"),+ Some(_) if app.runtime_connected() => (+ "●",+ Style::new().fg(ERR),+ "runtime connected, service reports offline",+ ),Some(_) => ("●", Style::new().fg(WARN), "paired, runtime not dialed in"),None if app.paired() => ("○", Style::new().fg(WARN), "paired, unreachable"),None => ("○", Style::new().fg(DIM), "not paired"),diff --git a/crates/harnesses/AGENTS.md b/crates/harnesses/AGENTS.mdindex bca58b4bee..9b95a86f8b 100644--- a/crates/harnesses/AGENTS.md+++ b/crates/harnesses/AGENTS.md@@ -39,6 +39,16 @@ daemon reconciles its webhook feed from.- `cargo test -p harnesses` (`SQLX_OFFLINE` unset).+## Gotchas++- `Harness.connected` reads `harness_runtime_connection`: true while a row has an+ unexpired lease. `agent_harness_service` writes those rows per connection. With+ no rows at all it falls back to `last_connected_at > last_disconnected_at`, for+ connections held by replicas older than the rows; remove that once none remain.+- `GET /harnesses/me` logs `harness_presence_disagreement` when the daemon sends+ `RUNTIME_CONNECTED_HEADER` and the harness reads offline+ (`infra/stacks/monitoring/monitors/harness-presence.ts` alerts on it).+## Related- Rules: `.agents/rules/rust/architecture.md`, `.agents/rules/rust/security.md`diff --git a/crates/harnesses/src/domain/models.rs b/crates/harnesses/src/domain/models.rsindex 1f6553697f..41e73c25c8 100644--- a/crates/harnesses/src/domain/models.rs+++ b/crates/harnesses/src/domain/models.rs@@ -55,6 +55,10 @@ pub struct Harness {pub last_connected_at: Option<DateTime<Utc>>,}+/// Sent by a daemon reading its own harness while it holds a runtime connection,+/// so the service can log when its own answer disagrees.+pub const RUNTIME_CONNECTED_HEADER: &str = "x-macro-harness-runtime-connected";+/// Request to open a pairing: the daemon asks for a code the user approves.////// The daemon serializes this, so both derives are used.diff --git a/crates/harnesses/src/inbound/axum_router.rs b/crates/harnesses/src/inbound/axum_router.rsindex f88456f3ec..0e67e59e4c 100644--- a/crates/harnesses/src/inbound/axum_router.rs+++ b/crates/harnesses/src/inbound/axum_router.rs@@ -5,7 +5,7 @@ use std::sync::Arc;use axum::{Json, Router,extract::{FromRef, Path, State},- http::StatusCode,+ http::{HeaderMap, StatusCode},response::{IntoResponse, Response},routing::{delete, get, patch, post},};@@ -22,11 +22,14 @@ use crate::domain::{models::{ApprovePairingRequest, ClaimOutcome, ClaimPairingRequest, ClaimedPairing,CreatePairingRequest, CreatedPairing, Harness, HarnessAgent, HarnessSession,- PairingDetails, PatchHarnessRequest,+ PairingDetails, PatchHarnessRequest, RUNTIME_CONNECTED_HEADER,},ports::{HarnessError, HarnessService},};+#[cfg(test)]+mod test;+/// State for the harnesses router.pub struct HarnessesRouterState<S, Auth> {service: Arc<S>,@@ -352,13 +355,29 @@ pub async fn delete_harness_handler<S: HarnessService, Auth: MacroAuthorizationSpub async fn get_self_harness_handler<S: HarnessService, Auth: MacroAuthorizationService>(State(state): State<HarnessesRouterState<S, Auth>>,authorization: MacroAuthorizationExtractor<Auth, HarnessOnly>,+ headers: HeaderMap,) -> Result<Json<Harness>, HarnessesHandlerErr> {- Ok(Json(- state- .service- .get_self(authorization.authorization.harness_id)- .await?,- ))+ let harness = state+ .service+ .get_self(authorization.authorization.harness_id)+ .await?;++ if runtime_presence_disagrees(&headers, &harness) {+ tracing::warn!(+ telemetry_event = "harness_presence_disagreement",+ harness = %harness.id,+ last_connected_at = ?harness.last_connected_at,+ "a daemon holds a runtime connection this harness reads as offline"+ );+ }++ Ok(Json(harness))+}++/// Whether the daemon asking says it holds a connection the harness reads as not+/// having. Only the daemon can see this: the service's own answer is what is wrong.+fn runtime_presence_disagrees(headers: &HeaderMap, harness: &Harness) -> bool {+ headers.contains_key(RUNTIME_CONNECTED_HEADER) && !harness.connected}/// Handler for `DELETE /harnesses/me`.diff --git a/crates/harnesses/src/inbound/axum_router/test.rs b/crates/harnesses/src/inbound/axum_router/test.rsnew file mode 100644index 0000000000..adc25c81bf--- /dev/null+++ b/crates/harnesses/src/inbound/axum_router/test.rs@@ -0,0 +1,35 @@+use axum::http::HeaderValue;+use chrono::Utc;++use super::*;+use crate::domain::models::HarnessOwner;++fn harness(connected: bool) -> Harness {+ Harness {+ id: HarnessId::TEST_A,+ allow_permission_bypass: false,+ kind: "macrod".to_owned(),+ name: "desk".to_owned(),+ owner: HarnessOwner::User {+ user_id: "macro|[email protected]".to_owned(),+ },+ created_by: "macro|[email protected]".to_owned(),+ created_at: Utc::now(),+ updated_at: Utc::now(),+ connected,+ last_connected_at: None,+ }+}++#[test]+fn only_a_daemon_holding_a_connection_can_disagree_with_offline() {+ let mut holding = HeaderMap::new();+ holding.insert(RUNTIME_CONNECTED_HEADER, HeaderValue::from_static("true"));++ assert!(runtime_presence_disagrees(&holding, &harness(false)));+ assert!(!runtime_presence_disagrees(&holding, &harness(true)));+ assert!(!runtime_presence_disagrees(+ &HeaderMap::new(),+ &harness(false)+ ));+}diff --git a/crates/harnesses/src/outbound/pg_harness_repo.rs b/crates/harnesses/src/outbound/pg_harness_repo.rsindex daa9eccbc1..d7ced98ddd 100644--- a/crates/harnesses/src/outbound/pg_harness_repo.rs+++ b/crates/harnesses/src/outbound/pg_harness_repo.rs@@ -44,22 +44,41 @@ struct HarnessRow {updated_at: DateTime<Utc>,last_connected_at: Option<DateTime<Utc>>,last_disconnected_at: Option<DateTime<Utc>>,+ has_live_connection: bool,+ has_connection_record: bool,+}++impl HarnessRow {+ /// A live connection row is the answer. A row whose lease lapsed means the+ /// replica holding it died. With no rows at all, the connection may come+ /// from a replica older than these rows, which only stamps the timestamps.+ fn connected(&self) -> bool {+ if self.has_live_connection {+ return true;+ }++ if self.has_connection_record {+ return false;+ }++ match (self.last_connected_at, self.last_disconnected_at) {+ (Some(connected_at), Some(disconnected_at)) => connected_at > disconnected_at,+ (Some(_), None) => true,+ (None, _) => false,+ }+ }}impl TryFrom<HarnessRow> for Harness {type Error = anyhow::Error;fn try_from(row: HarnessRow) -> Result<Self, Self::Error> {+ let connected = row.connected();let owner = match (row.owner_user_id, row.team_id) {(Some(user_id), None) => HarnessOwner::User { user_id },(None, Some(team_id)) => HarnessOwner::Team { team_id },_ => anyhow::bail!("harness row violated its owner invariant"),};- let connected = match (row.last_connected_at, row.last_disconnected_at) {- (Some(connected_at), Some(disconnected_at)) => connected_at > disconnected_at,- (Some(_), None) => true,- (None, _) => false,- };Ok(Self {id: HarnessId::new_from_uuid(row.id),@@ -94,7 +113,15 @@ async fn fetch_harness(r#"SELECTid, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,- created_at, updated_at, last_connected_at, last_disconnected_at+ created_at, updated_at, last_connected_at, last_disconnected_at,+ EXISTS (+ SELECT 1 FROM harness_runtime_connection c+ WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()+ ) AS "has_live_connection!",+ EXISTS (+ SELECT 1 FROM harness_runtime_connection c+ WHERE c.harness_id = harnesses.id+ ) AS "has_connection_record!"FROM harnessesWHERE id = $1 AND deleted_at IS NULL"#,@@ -338,7 +365,15 @@ impl HarnessRepo for PgHarnessRepo {r#"SELECTid, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,- created_at, updated_at, last_connected_at, last_disconnected_at+ created_at, updated_at, last_connected_at, last_disconnected_at,+ EXISTS (+ SELECT 1 FROM harness_runtime_connection c+ WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()+ ) AS "has_live_connection!",+ EXISTS (+ SELECT 1 FROM harness_runtime_connection c+ WHERE c.harness_id = harnesses.id+ ) AS "has_connection_record!"FROM harnessesWHERE deleted_at IS NULLAND (@@ -376,7 +411,15 @@ impl HarnessRepo for PgHarnessRepo {WHERE id = $1 AND deleted_at IS NULLRETURNINGid, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,- created_at, updated_at, last_connected_at, last_disconnected_at+ created_at, updated_at, last_connected_at, last_disconnected_at,+ EXISTS (+ SELECT 1 FROM harness_runtime_connection c+ WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()+ ) AS "has_live_connection!",+ EXISTS (+ SELECT 1 FROM harness_runtime_connection c+ WHERE c.harness_id = harnesses.id+ ) AS "has_connection_record!""#,harness_id.as_uuid(),changes.name,diff --git a/crates/harnesses/src/outbound/pg_harness_repo/tests.rs b/crates/harnesses/src/outbound/pg_harness_repo/tests.rsindex 14238e0805..8f67f61f0f 100644--- a/crates/harnesses/src/outbound/pg_harness_repo/tests.rs+++ b/crates/harnesses/src/outbound/pg_harness_repo/tests.rs@@ -281,7 +281,7 @@ async fn deletion_soft_deletes_and_revokes_tokens(pool: PgPool) {}#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]-async fn connected_state_follows_the_presence_timestamps(pool: PgPool) {+async fn connected_state_falls_back_to_timestamps_without_connection_rows(pool: PgPool) {let repo = PgHarnessRepo::new(pool.clone());let harness_id = Uuid::new_v4();sqlx::query!(@@ -308,6 +308,62 @@ async fn connected_state_follows_the_presence_timestamps(pool: PgPool) {assert!(!harness.connected);}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn connected_state_follows_connection_leases_over_timestamps(pool: PgPool) {+ let repo = PgHarnessRepo::new(pool.clone());+ let harness_id = Uuid::new_v4();+ sqlx::query!(+ r#"+ INSERT INTO harnesses (id, name, owner_user_id, created_by, last_connected_at, last_disconnected_at)+ VALUES ($1, 'mine', $2, $2, now(), now() + interval '1 hour')+ "#,+ harness_id,+ OWNER_ID,+ )+ .execute(&pool)+ .await+ .unwrap();+ let connection_id = Uuid::new_v4();+ sqlx::query!(+ r#"+ INSERT INTO harness_runtime_connection (harness_id, id, replica_id, lease_expires_at)+ VALUES ($1, $2, $3, now() + interval '30 seconds')+ "#,+ harness_id,+ connection_id,+ Uuid::new_v4(),+ )+ .execute(&pool)+ .await+ .unwrap();+ let harness_id = HarnessId::new_from_uuid(harness_id);++ // A live lease wins over a later disconnect stamp.+ let harness = repo.get_harness(harness_id).await.unwrap().unwrap();+ assert!(harness.connected);++ sqlx::query!(+ r#"+ UPDATE harnesses SET last_disconnected_at = NULL;+ "#,+ )+ .execute(&pool)+ .await+ .unwrap();+ sqlx::query!(+ r#"UPDATE harness_runtime_connection SET lease_expires_at = now() - interval '1 second'"#,+ )+ .execute(&pool)+ .await+ .unwrap();++ // A crashed replica writes no disconnect; the lapsed lease is what ends it.+ let harness = repo.get_harness(harness_id).await.unwrap().unwrap();+ assert!(!harness.connected);+ let listed = repo.list_visible_harnesses(caller()).await.unwrap();+ assert!(!listed.iter().any(|harness| harness.connected));+}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn bound_agents_lists_only_live_agents_of_this_harness(pool: PgPool) {let repo = PgHarnessRepo::new(pool.clone());diff --git a/crates/macro_db_client/migrations/20261105153310_create_harness_runtime_connections.sql b/crates/macro_db_client/migrations/20261105153310_create_harness_runtime_connections.sqlnew file mode 100644index 0000000000..095dc264df--- /dev/null+++ b/crates/macro_db_client/migrations/20261105153310_create_harness_runtime_connections.sql@@ -0,0 +1,13 @@+-- One row per live harness runtime connection, written by the replica holding+-- its socket. A harness is connected while a row with an unexpired lease exists,+-- so one connection's close cannot affect another's, whatever order replicas+-- notice them in, and a replica that crashes takes its connections offline when+-- its leases lapse.+CREATE TABLE public.harness_runtime_connection (+ harness_id uuid NOT NULL REFERENCES public.harnesses(id) ON DELETE CASCADE,+ id uuid NOT NULL,+ replica_id uuid NOT NULL,+ connected_at timestamptz NOT NULL DEFAULT now(),+ lease_expires_at timestamptz NOT NULL,+ PRIMARY KEY (harness_id, id)+);diff --git a/infra/stacks/monitoring/AGENTS.md b/infra/stacks/monitoring/AGENTS.mdindex c7b8d3a759..7f8418c331 100644--- a/infra/stacks/monitoring/AGENTS.md+++ b/infra/stacks/monitoring/AGENTS.md@@ -61,6 +61,8 @@ From `infra/`:- `monitors/whatsapp-session.ts` reads log events by `telemetry_event`; itsheader is the field contract with `services/whatsapp_session_service`.Renaming a field there silently turns its monitor into missing data.+- `monitors/harness-presence.ts` reads `harness_presence_disagreement` logged by+ `crates/harnesses/src/inbound/axum_router.rs`; its header is the field contract.- `monitors/outlook-sync.ts` reads the health log, sync spans and Graph andtoken span fields email-service emits; its header is the field contract.It finds the Graph and token spans by `@outlook.throttle_origin` anddiff --git a/infra/stacks/monitoring/README.md b/infra/stacks/monitoring/README.mdindex aa9c24e694..b0fe6ea039 100644--- a/infra/stacks/monitoring/README.md+++ b/infra/stacks/monitoring/README.md@@ -115,8 +115,9 @@ The gate is that preview:of the channel), and the GraphQL soup volume, websocket upgrade, gatewayconnections and task count monitors their `query` and thresholds (volumeand websocket upgrades also gain recovery thresholds).-- **90 resources to create**: 70 monitors (31 prod threshold and anomaly, 19 GraphQL soup, 9- notification retention, 10 WhatsApp session, 1 native call push), 2 webhooks,+- **92 resources to create**: 72 monitors (31 prod threshold and anomaly, 19 GraphQL soup, 9+ notification retention, 10 WhatsApp session, 1 native call push, 2 harness+ presence), 2 webhooks,7 log-based metrics, 3 span-based metrics and 1 APM retention filter (6, 3and 1 of them for Outlook sync), the GoAlert synthetic test, the five prodalert dashboards, and the notification retention dashboard. If the pagingdiff --git a/infra/stacks/monitoring/index.ts b/infra/stacks/monitoring/index.tsindex bad01779e0..e514ba60f7 100644--- a/infra/stacks/monitoring/index.ts+++ b/infra/stacks/monitoring/index.ts@@ -21,6 +21,7 @@ import './goalert';import './monitors/ai-editing';import './monitors/calls';import './monitors/graphql-soup';+import './monitors/harness-presence';import './monitors/macrodb';import './monitors/outlook-sync';import './monitors/queues';diff --git a/infra/stacks/monitoring/monitors/harness-presence.ts b/infra/stacks/monitoring/monitors/harness-presence.tsnew file mode 100644index 0000000000..bf66ec23c8--- /dev/null+++ b/infra/stacks/monitoring/monitors/harness-presence.ts@@ -0,0 +1,41 @@+// Harness presence: whether the service's "connected" agrees with macrod.+//+// New monitor, not adopted. It reads a structured log event, keyed by+// `telemetry_event`, so the fields below are its contract:+//+// - `harness_presence_disagreement`, from document-storage-service, at most once+// a minute per harness: macrod says it holds a runtime connection and the+// service reads that harness as offline. Fields: `harness`.+//+// Ids are opaque harness uuids. Macrod sends the signal only once its connection+// has been up for a full minute, so a healthy reconnect never reaches it.+import * as datadog from '@pulumi/datadog';+import { goalertWebhooks } from '../goalert';+import { notify, RENOTIFY_MINUTES } from '../notify';++for (const environment of ['dev', 'prod'] as const) {+ const env = `env:${environment}`;++ new datadog.Monitor(+ `harness-presence-disagreement-${environment}`,+ {+ name: `[${environment.toUpperCase()}] Harness presence: {{harness.name}} online but reported offline`,+ type: 'log alert',+ query: `logs("@telemetry_event:harness_presence_disagreement ${env}").index("*").rollup("count").by("@harness").last("15m") > 5`,+ message: `Macrod for harness {{harness.name}} has held a runtime connection for at least five minutes while the service reports it offline, so the app marks its agents unavailable.++Search @harness:{{harness.name}} @telemetry_event:(harness_runtime_connected OR harness_runtime_disconnected OR harness_runtime_lease_restored). A live connection with no row, or a lease that stopped renewing, is on the replica named in those events. See crates/agent_harness/AGENTS.md.++${notify({ pages: false, environment })}`,+ monitorThresholds: { critical: '5' },+ tags: [env, 'service:agent-harness-service'],+ priority: environment === 'prod' ? '2' : '3',+ includeTags: true,+ requireFullWindow: false,+ renotifyInterval: RENOTIFY_MINUTES,+ // Daemons older than this signal send nothing; that is not a disagreement.+ onMissingData: 'resolve',+ },+ { dependsOn: goalertWebhooks }+ );+}diff --git a/services/agent_harness_service/src/harness_bindings.rs b/services/agent_harness_service/src/harness_bindings.rsindex dbda857a7a..33051b5f55 100644--- a/services/agent_harness_service/src/harness_bindings.rs+++ b/services/agent_harness_service/src/harness_bindings.rs@@ -3,22 +3,34 @@//! Composition-root adapters: [`PgHarnessBindings`] answers "which registered//! harness serves this bot right now" for session binding, and//! [`PgHarnessPresence`] writes the attach/detach bookkeeping the harness-//! settings page reads connection state from.+//! settings page reads connection state from: one leased row per connection, so+//! replicas that see a harness's sockets end out of order cannot overwrite each+//! other's answer.use std::future::Future;use std::pin::Pin;use std::sync::Arc;+use std::time::Duration;use agent_client_protocol::schema::v1::SessionId;-use agent_harness::domain::ports::{HarnessBindings, HarnessPresence};-use agent_session::domain::model::AgentSessionId;+use agent_harness::domain::ports::{HarnessBindings, HarnessPresence, RuntimeConnectionId};+use agent_session::domain::model::{AgentSessionId, ReplicaId};use bot_id::BotId;use harness_id::HarnessId;use sqlx::PgPool;+use tracing::Instrument as _;#[cfg(test)]mod test;+/// How long a connection counts as live without a renewal. Three missed+/// renewals: a slow write does not flip a harness offline, and a crashed+/// replica's connections end within half a minute.+const RUNTIME_CONNECTION_LEASE: Duration = Duration::from_secs(30);++/// How often a replica renews the leases of the connections it holds.+pub const RUNTIME_CONNECTION_RENEW_EVERY: Duration = Duration::from_secs(10);+/// [`HarnessBindings`] over the `agent_configs` table.#[derive(Clone)]pub struct PgHarnessBindings {@@ -78,46 +90,209 @@ impl HarnessBindings for PgHarnessBindings {}}-/// [`HarnessPresence`] over the `harnesses` table.+/// Take a harness's presence lock until the transaction ends.+///+/// Closing decides whether it was the last live connection by reading its peers'+/// rows. Without this, two simultaneous closes each count the other as live and+/// neither stamps the harness. `NO KEY` leaves renewals' foreign key checks free.+async fn lock_presence(+ transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,+ harness: HarnessId,+) -> sqlx::Result<()> {+ sqlx::query_scalar!(+ r#"SELECT id FROM harnesses WHERE id = $1 FOR NO KEY UPDATE"#,+ harness.as_uuid(),+ )+ .fetch_optional(&mut **transaction)+ .await?;+ Ok(())+}++/// [`HarnessPresence`] over the `harness_runtime_connection` table, kept in step+/// with the `harnesses` presence timestamps.pub struct PgHarnessPresence {pool: PgPool,+ replica: ReplicaId,}impl PgHarnessPresence {- /// Wrap a pool.- pub fn new(pool: PgPool) -> Self {- Self { pool }+ /// Wrap a pool, writing as `replica`.+ pub fn new(pool: PgPool, replica: ReplicaId) -> Self {+ Self { pool, replica }}}impl HarnessPresence for PgHarnessPresence {- fn connected(self: Arc<Self>, harness: HarnessId) -> Pin<Box<dyn Future<Output = ()> + Send>> {- Box::pin(async move {- let result = sqlx::query!(- r#"UPDATE harnesses SET last_connected_at = now() WHERE id = $1"#,- harness.as_uuid(),- )- .execute(&self.pool)- .await;- if let Err(error) = result {- tracing::error!(error = ?error, %harness, "failed to record harness attach");+ fn connected(+ self: Arc<Self>,+ harness: HarnessId,+ connection: RuntimeConnectionId,+ ) -> Pin<Box<dyn Future<Output = ()> + Send>> {+ let span = tracing::info_span!(+ "harness.presence.connected",+ %harness,+ %connection,+ replica = %self.replica,+ );+ Box::pin(+ async move {+ // Lapsed rows go here rather than on a timer: until a harness+ // dials again they are what says its last replica died.+ let result = async {+ let mut transaction = self.pool.begin().await?;+ lock_presence(&mut transaction, harness).await?;+ sqlx::query!(+ r#"+ WITH reaped AS (+ DELETE FROM harness_runtime_connection+ WHERE harness_id = $1 AND lease_expires_at <= now()+ ), added AS (+ INSERT INTO harness_runtime_connection+ (harness_id, id, replica_id, lease_expires_at)+ VALUES ($1, $2, $3, now() + make_interval(secs => $4))+ ON CONFLICT (harness_id, id) DO NOTHING+ )+ UPDATE harnesses SET last_connected_at = now() WHERE id = $1+ "#,+ harness.as_uuid(),+ connection.as_uuid(),+ self.replica.as_uuid(),+ RUNTIME_CONNECTION_LEASE.as_secs_f64(),+ )+ .execute(&mut *transaction)+ .await?;+ transaction.commit().await+ }+ .await;+ match result {+ Ok(()) => tracing::info!(+ telemetry_event = "harness_runtime_connected",+ %harness,+ %connection,+ replica = %self.replica,+ "recorded a harness runtime connection"+ ),+ Err(error) => {+ tracing::error!(error = ?error, %harness, %connection, "failed to record harness attach");+ }+ }}- })+ .instrument(span),+ )}fn disconnected(self: Arc<Self>,harness: HarnessId,+ connection: RuntimeConnectionId,+ ) -> Pin<Box<dyn Future<Output = ()> + Send>> {+ let span = tracing::info_span!(+ "harness.presence.disconnected",+ %harness,+ %connection,+ replica = %self.replica,+ );+ Box::pin(+ async move {+ // The stamp is what a reader without these rows goes by, so it+ // moves only when this was the harness's last live connection.+ let result = async {+ let mut transaction = self.pool.begin().await?;+ lock_presence(&mut transaction, harness).await?;+ let closed = sqlx::query!(+ r#"+ WITH removed AS (+ DELETE FROM harness_runtime_connection+ WHERE harness_id = $1 AND (id = $2 OR lease_expires_at <= now())+ RETURNING id, connected_at+ ), others AS (+ SELECT count(*) AS live+ FROM harness_runtime_connection+ WHERE harness_id = $1 AND id <> $2 AND lease_expires_at > now()+ ), stamped AS (+ UPDATE harnesses SET last_disconnected_at = now()+ WHERE id = $1 AND (SELECT live FROM others) = 0+ )+ SELECT+ (SELECT live FROM others) AS "other_live_connections!",+ (+ SELECT (extract(epoch FROM now() - connected_at) * 1000)::bigint+ FROM removed+ WHERE id = $2+ ) AS connected_for_ms+ "#,+ harness.as_uuid(),+ connection.as_uuid(),+ )+ .fetch_one(&mut *transaction)+ .await?;+ transaction.commit().await?;+ Ok::<_, sqlx::Error>(closed)+ }+ .await;+ match result {+ Ok(closed) => tracing::info!(+ telemetry_event = "harness_runtime_disconnected",+ %harness,+ %connection,+ replica = %self.replica,+ other_live_connections = closed.other_live_connections,+ connected_for_ms = closed.connected_for_ms,+ "recorded a closed harness runtime connection"+ ),+ Err(error) => {+ tracing::error!(error = ?error, %harness, %connection, "failed to record harness detach");+ }+ }+ }+ .instrument(span),+ )+ }++ fn renew(+ self: Arc<Self>,+ live: Vec<(HarnessId, RuntimeConnectionId)>,) -> Pin<Box<dyn Future<Output = ()> + Send>> {Box::pin(async move {- let result = sqlx::query!(- r#"UPDATE harnesses SET last_disconnected_at = now() WHERE id = $1"#,- harness.as_uuid(),+ let (harnesses, connections): (Vec<_>, Vec<_>) = live+ .into_iter()+ .map(|(harness, connection)| (harness.as_uuid(), connection.as_uuid()))+ .unzip();+ // An upsert, so a connection whose attach write failed, or whose+ // lapsed row was reaped while this process stalled, comes back.+ let renewed = sqlx::query!(+ r#"+ INSERT INTO harness_runtime_connection+ (harness_id, id, replica_id, lease_expires_at)+ SELECT live.harness_id, live.id, $3, now() + make_interval(secs => $4)+ FROM UNNEST($1::uuid[], $2::uuid[]) AS live(harness_id, id)+ JOIN harnesses ON harnesses.id = live.harness_id+ ON CONFLICT (harness_id, id) DO UPDATE+ SET lease_expires_at = EXCLUDED.lease_expires_at+ RETURNING harness_id, id, (xmax = 0) AS "restored!"+ "#,+ &harnesses,+ &connections,+ self.replica.as_uuid(),+ RUNTIME_CONNECTION_LEASE.as_secs_f64(),)- .execute(&self.pool)+ .fetch_all(&self.pool).await;- if let Err(error) = result {- tracing::error!(error = ?error, %harness, "failed to record harness detach");+ match renewed {+ Ok(renewed) => {+ for row in renewed.into_iter().filter(|row| row.restored) {+ tracing::warn!(+ telemetry_event = "harness_runtime_lease_restored",+ harness = %row.harness_id,+ connection = %row.id,+ replica = %self.replica,+ "a held harness runtime connection had no presence row"+ );+ }+ }+ Err(error) => {+ tracing::warn!(error = ?error, replica = %self.replica, "failed to renew harness runtime leases");+ }}})}diff --git a/services/agent_harness_service/src/harness_bindings/test.rs b/services/agent_harness_service/src/harness_bindings/test.rsindex 021355c765..afec118e0f 100644--- a/services/agent_harness_service/src/harness_bindings/test.rs+++ b/services/agent_harness_service/src/harness_bindings/test.rs@@ -1,4 +1,6 @@use super::*;+use harnesses::domain::ports::HarnessRepo as _;+use harnesses::outbound::pg_harness_repo::PgHarnessRepo;use macro_db_migrator::MACRO_DB_MIGRATIONS;use sqlx::types::Uuid;@@ -113,3 +115,136 @@ async fn a_deleted_harness_reattaches_nothing(pool: PgPool) {assert!(found.is_empty());}++fn presence_on_a_replica(pool: &PgPool) -> Arc<PgHarnessPresence> {+ Arc::new(PgHarnessPresence::new(pool.clone(), ReplicaId::mint()))+}++async fn reads_connected(pool: &PgPool, harness: HarnessId) -> bool {+ PgHarnessRepo::new(pool.clone())+ .get_harness(harness)+ .await+ .unwrap()+ .unwrap()+ .connected+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_stale_close_on_another_replica_leaves_a_redialed_harness_connected(pool: PgPool) {+ let harness = insert_harness(&pool).await;+ let replica_a = presence_on_a_replica(&pool);+ let replica_b = presence_on_a_replica(&pool);+ let old_socket = RuntimeConnectionId::generate();+ let new_socket = RuntimeConnectionId::generate();++ Arc::clone(&replica_a).connected(harness, old_socket).await;+ // The old socket goes silent. The daemon redials, reaching the other replica+ // before the first one notices.+ Arc::clone(&replica_b).connected(harness, new_socket).await;+ assert!(reads_connected(&pool, harness).await);++ replica_a.disconnected(harness, old_socket).await;++ assert!(reads_connected(&pool, harness).await);++ // The stamp stays put too, for readers that still go by the timestamps.+ let stamped = sqlx::query_scalar!(+ r#"SELECT last_disconnected_at FROM harnesses WHERE id = $1"#,+ harness.as_uuid(),+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(stamped, None);++ replica_b.disconnected(harness, new_socket).await;++ assert!(!reads_connected(&pool, harness).await);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_replica_crash_ends_its_connections_when_their_lease_lapses(pool: PgPool) {+ let harness = insert_harness(&pool).await;+ let crashed = presence_on_a_replica(&pool);+ let connection = RuntimeConnectionId::generate();+ crashed.connected(harness, connection).await;+ assert!(reads_connected(&pool, harness).await);++ // A crash writes no disconnect; all that happens is renewals stop.+ sqlx::query!(+ r#"UPDATE harness_runtime_connection SET lease_expires_at = now() - interval '1 second'"#,+ )+ .execute(&pool)+ .await+ .unwrap();+ assert!(!reads_connected(&pool, harness).await);++ // The daemon's next dial brings it back and clears the dead row.+ let successor = presence_on_a_replica(&pool);+ successor+ .connected(harness, RuntimeConnectionId::generate())+ .await;+ assert!(reads_connected(&pool, harness).await);+ let rows = sqlx::query_scalar!(+ r#"SELECT count(*) AS "count!" FROM harness_runtime_connection WHERE harness_id = $1"#,+ harness.as_uuid(),+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(rows, 1);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn renewal_extends_a_lease_and_restores_a_missing_row(pool: PgPool) {+ let harness = insert_harness(&pool).await;+ let replica = presence_on_a_replica(&pool);+ let connection = RuntimeConnectionId::generate();+ Arc::clone(&replica).connected(harness, connection).await;++ sqlx::query!(+ r#"UPDATE harness_runtime_connection SET lease_expires_at = now() - interval '1 second'"#,+ )+ .execute(&pool)+ .await+ .unwrap();+ Arc::clone(&replica)+ .renew(vec![(harness, connection)])+ .await;+ assert!(reads_connected(&pool, harness).await);++ sqlx::query!(r#"DELETE FROM harness_runtime_connection"#)+ .execute(&pool)+ .await+ .unwrap();+ sqlx::query!(r#"UPDATE harnesses SET last_disconnected_at = now() + interval '1 hour'"#)+ .execute(&pool)+ .await+ .unwrap();+ assert!(!reads_connected(&pool, harness).await);++ replica.renew(vec![(harness, connection)]).await;++ assert!(reads_connected(&pool, harness).await);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn simultaneous_final_closes_leave_a_harness_offline(pool: PgPool) {+ // Racing writes interleave differently each time, so give them many chances.+ for _ in 0..25 {+ let harness = insert_harness(&pool).await;+ let replica_a = presence_on_a_replica(&pool);+ let replica_b = presence_on_a_replica(&pool);+ let first = RuntimeConnectionId::generate();+ let second = RuntimeConnectionId::generate();+ Arc::clone(&replica_a).connected(harness, first).await;+ Arc::clone(&replica_b).connected(harness, second).await;++ tokio::join!(+ replica_a.disconnected(harness, first),+ replica_b.disconnected(harness, second),+ );++ assert!(!reads_connected(&pool, harness).await);+ }+}diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rsindex 36c2aff2ef..81aee2acba 100644--- a/services/agent_harness_service/src/main.rs+++ b/services/agent_harness_service/src/main.rs@@ -136,7 +136,7 @@ use github::outbound::pull_request_diff::GithubPullRequestDiffClient;use github_pull_requests::domain::service::GithubPullRequestChangesetStore;use github_pull_requests::outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo;use github_pull_requests::outbound::s3_patch_store::S3GithubPullRequestPatchStore;-use harness_bindings::{PgHarnessBindings, PgHarnessPresence};+use harness_bindings::{PgHarnessBindings, PgHarnessPresence, RUNTIME_CONNECTION_RENEW_EVERY};use harnesses::outbound::pg_harness_repo::PgHarnessRepo;use kafka_util::{GroupName, KafkaEventConsumer, consumer_span, record_span_error};use lexical_client::LexicalClient;@@ -1283,9 +1283,10 @@ async fn run() -> anyhow::Result<()> {// the harness loads once it exists below.let (live_sessions, mut live_session_reports) = tokio::sync::mpsc::unbounded_channel();let runtimes = RuntimeRegistry::with_presence(- Arc::new(PgHarnessPresence::new(pool.clone())),+ Arc::new(PgHarnessPresence::new(pool.clone(), replica)),live_sessions,);+ let presence_runtimes = Arc::clone(&runtimes);// Read once, here, rather than at every session this deployment opens: a// URL that names no repository is a misconfiguration of the deployment,// and refusing it at startup is the difference between one loud failure@@ -1881,6 +1882,18 @@ async fn run() -> anyhow::Result<()> {}});+ // Keeps vouching for the runtime connections this process holds, including+ // while it drains: they stay live until their sockets close. When this stops+ // in a crash, the leases lapse and those harnesses read offline.+ let presence_renewal = tokio::spawn(async move {+ let mut ticker = tokio::time::interval(RUNTIME_CONNECTION_RENEW_EVERY);+ ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);+ loop {+ ticker.tick().await;+ presence_runtimes.renew_presence().await;+ }+ });+// Viewers drop a typing indicator nobody refreshes after a few seconds;// every turn running here keeps its agent's indicator alive until it ends.let typing_harness = harness.clone();@@ -2260,6 +2273,7 @@ async fn run() -> anyhow::Result<()> {trigger.abort();egress_http.abort();heartbeat.abort();+ presence_renewal.abort();dm_recovery.abort();typing_heartbeat.abort();recovery.abort();