Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(harnesses): stop a stale socket close marking a live harness offline (#9872)

Eric Hayes776ca7854bc2
diff --git a/.sqlx/query-d30ffc9d1a8aabb56674f4a0292176684576a144a1dd05b4e6fed48f4e49e3a9.json b/.sqlx/query-1c08845abe8781e7beb18b8dbf5e367cf3b95acc7a26029a553efd8e2ca2b7a9.json
similarity index 60%
rename from .sqlx/query-d30ffc9d1a8aabb56674f4a0292176684576a144a1dd05b4e6fed48f4e49e3a9.json
rename to .sqlx/query-1c08845abe8781e7beb18b8dbf5e367cf3b95acc7a26029a553efd8e2ca2b7a9.json
index ad20fc9231..1036e6b7be 100644
--- a/.sqlx/query-d30ffc9d1a8aabb56674f4a0292176684576a144a1dd05b4e6fed48f4e49e3a9.json
+++ b/.sqlx/query-1c08845abe8781e7beb18b8dbf5e367cf3b95acc7a26029a553efd8e2ca2b7a9.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n SELECT\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at\n FROM harnesses\n WHERE deleted_at IS NULL\n AND (\n owner_user_id = $1\n OR team_id IN (SELECT team_id FROM team_user WHERE user_id = $1)\n )\n ORDER BY created_at DESC\n ",
+ "query": "\n SELECT\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at,\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()\n ) AS \"has_live_connection!\",\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id\n ) AS \"has_connection_record!\"\n FROM harnesses\n WHERE deleted_at IS NULL\n AND (\n owner_user_id = $1\n OR team_id IN (SELECT team_id FROM team_user WHERE user_id = $1)\n )\n ORDER BY created_at DESC\n ",
"describe": {
"columns": [
{
@@ -57,6 +57,16 @@
"ordinal": 10,
"name": "last_disconnected_at",
"type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 11,
+ "name": "has_live_connection!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 12,
+ "name": "has_connection_record!",
+ "type_info": "Bool"
}
],
"parameters": {
@@ -75,8 +85,10 @@
false,
false,
true,
- true
+ true,
+ null,
+ null
]
},
- "hash": "d30ffc9d1a8aabb56674f4a0292176684576a144a1dd05b4e6fed48f4e49e3a9"
+ "hash": "1c08845abe8781e7beb18b8dbf5e367cf3b95acc7a26029a553efd8e2ca2b7a9"
}
diff --git a/.sqlx/query-65324cc6bff4a47702524a0ffc2b2e12acbd283bb0e90732e64cb74ad430ebcb.json b/.sqlx/query-65324cc6bff4a47702524a0ffc2b2e12acbd283bb0e90732e64cb74ad430ebcb.json
new file mode 100644
index 0000000000..a8c618ecec
--- /dev/null
+++ b/.sqlx/query-65324cc6bff4a47702524a0ffc2b2e12acbd283bb0e90732e64cb74ad430ebcb.json
@@ -0,0 +1,29 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH removed AS (\n DELETE FROM harness_runtime_connection\n WHERE harness_id = $1 AND (id = $2 OR lease_expires_at <= now())\n RETURNING id, connected_at\n ), others AS (\n SELECT count(*) AS live\n FROM harness_runtime_connection\n WHERE harness_id = $1 AND id <> $2 AND lease_expires_at > now()\n ), stamped AS (\n UPDATE harnesses SET last_disconnected_at = now()\n WHERE id = $1 AND (SELECT live FROM others) = 0\n )\n SELECT\n (SELECT live FROM others) AS \"other_live_connections!\",\n (\n SELECT (extract(epoch FROM now() - connected_at) * 1000)::bigint\n FROM removed\n WHERE id = $2\n ) AS connected_for_ms\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "other_live_connections!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "connected_for_ms",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ null,
+ null
+ ]
+ },
+ "hash": "65324cc6bff4a47702524a0ffc2b2e12acbd283bb0e90732e64cb74ad430ebcb"
+}
diff --git a/.sqlx/query-65ad06326d16861dbcce0f6d6c0a6e3459cb21197ed70222a139bbfc727e9c0d.json b/.sqlx/query-65ad06326d16861dbcce0f6d6c0a6e3459cb21197ed70222a139bbfc727e9c0d.json
new file mode 100644
index 0000000000..109c658fa3
--- /dev/null
+++ b/.sqlx/query-65ad06326d16861dbcce0f6d6c0a6e3459cb21197ed70222a139bbfc727e9c0d.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT id FROM harnesses WHERE id = $1 FOR NO KEY UPDATE",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "65ad06326d16861dbcce0f6d6c0a6e3459cb21197ed70222a139bbfc727e9c0d"
+}
diff --git a/.sqlx/query-19c6aed4e629ba1a98572498d7803e1027d17a7215beb36d0e3b3c54cb635396.json b/.sqlx/query-88f86058dc56cda26b0c2be64b6b2e183027fc2dae082a901caa6c40c1d89d93.json
similarity index 70%
rename from .sqlx/query-19c6aed4e629ba1a98572498d7803e1027d17a7215beb36d0e3b3c54cb635396.json
rename to .sqlx/query-88f86058dc56cda26b0c2be64b6b2e183027fc2dae082a901caa6c40c1d89d93.json
index 43158bb9c7..b69af0ee32 100644
--- a/.sqlx/query-19c6aed4e629ba1a98572498d7803e1027d17a7215beb36d0e3b3c54cb635396.json
+++ b/.sqlx/query-88f86058dc56cda26b0c2be64b6b2e183027fc2dae082a901caa6c40c1d89d93.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n UPDATE harnesses\n SET\n name = COALESCE($2, name),\n allow_permission_bypass = COALESCE($3, allow_permission_bypass),\n updated_at = now()\n WHERE id = $1 AND deleted_at IS NULL\n RETURNING\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at\n ",
+ "query": "\n UPDATE harnesses\n SET\n name = COALESCE($2, name),\n allow_permission_bypass = COALESCE($3, allow_permission_bypass),\n updated_at = now()\n WHERE id = $1 AND deleted_at IS NULL\n RETURNING\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at,\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()\n ) AS \"has_live_connection!\",\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id\n ) AS \"has_connection_record!\"\n ",
"describe": {
"columns": [
{
@@ -57,6 +57,16 @@
"ordinal": 10,
"name": "last_disconnected_at",
"type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 11,
+ "name": "has_live_connection!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 12,
+ "name": "has_connection_record!",
+ "type_info": "Bool"
}
],
"parameters": {
@@ -77,8 +87,10 @@
false,
false,
true,
- true
+ true,
+ null,
+ null
]
},
- "hash": "19c6aed4e629ba1a98572498d7803e1027d17a7215beb36d0e3b3c54cb635396"
+ "hash": "88f86058dc56cda26b0c2be64b6b2e183027fc2dae082a901caa6c40c1d89d93"
}
diff --git a/.sqlx/query-b4d134196b124c4c4a6b0ca5ceb6c8b0c14fbe1d11ad9c260c9108a27f64134d.json b/.sqlx/query-b4d134196b124c4c4a6b0ca5ceb6c8b0c14fbe1d11ad9c260c9108a27f64134d.json
deleted file mode 100644
index 7ce5d5b8f8..0000000000
--- a/.sqlx/query-b4d134196b124c4c4a6b0ca5ceb6c8b0c14fbe1d11ad9c260c9108a27f64134d.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "UPDATE harnesses SET last_disconnected_at = now() WHERE id = $1",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid"
- ]
- },
- "nullable": []
- },
- "hash": "b4d134196b124c4c4a6b0ca5ceb6c8b0c14fbe1d11ad9c260c9108a27f64134d"
-}
diff --git a/.sqlx/query-9c0a40ee4c671cdcd35dde8f6ef8278e3fc0a72e3f3ac82759719d660b4e4e5d.json b/.sqlx/query-c5052e4731eb9c665326a8884c1776feefaf57d261a5fe18c4a885feb4ffeed0.json
similarity index 65%
rename from .sqlx/query-9c0a40ee4c671cdcd35dde8f6ef8278e3fc0a72e3f3ac82759719d660b4e4e5d.json
rename to .sqlx/query-c5052e4731eb9c665326a8884c1776feefaf57d261a5fe18c4a885feb4ffeed0.json
index 26b3b367ae..0716c53614 100644
--- a/.sqlx/query-9c0a40ee4c671cdcd35dde8f6ef8278e3fc0a72e3f3ac82759719d660b4e4e5d.json
+++ b/.sqlx/query-c5052e4731eb9c665326a8884c1776feefaf57d261a5fe18c4a885feb4ffeed0.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n SELECT\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at\n FROM harnesses\n WHERE id = $1 AND deleted_at IS NULL\n ",
+ "query": "\n SELECT\n id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,\n created_at, updated_at, last_connected_at, last_disconnected_at,\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()\n ) AS \"has_live_connection!\",\n EXISTS (\n SELECT 1 FROM harness_runtime_connection c\n WHERE c.harness_id = harnesses.id\n ) AS \"has_connection_record!\"\n FROM harnesses\n WHERE id = $1 AND deleted_at IS NULL\n ",
"describe": {
"columns": [
{
@@ -57,6 +57,16 @@
"ordinal": 10,
"name": "last_disconnected_at",
"type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 11,
+ "name": "has_live_connection!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 12,
+ "name": "has_connection_record!",
+ "type_info": "Bool"
}
],
"parameters": {
@@ -75,8 +85,10 @@
false,
false,
true,
- true
+ true,
+ null,
+ null
]
},
- "hash": "9c0a40ee4c671cdcd35dde8f6ef8278e3fc0a72e3f3ac82759719d660b4e4e5d"
+ "hash": "c5052e4731eb9c665326a8884c1776feefaf57d261a5fe18c4a885feb4ffeed0"
}
diff --git a/.sqlx/query-de149836f561243b3afb7c1b1b8f74e79f1a89a7097a81754ceb38ba85008990.json b/.sqlx/query-de149836f561243b3afb7c1b1b8f74e79f1a89a7097a81754ceb38ba85008990.json
new file mode 100644
index 0000000000..539f6fe274
--- /dev/null
+++ b/.sqlx/query-de149836f561243b3afb7c1b1b8f74e79f1a89a7097a81754ceb38ba85008990.json
@@ -0,0 +1,17 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH reaped AS (\n DELETE FROM harness_runtime_connection\n WHERE harness_id = $1 AND lease_expires_at <= now()\n ), added AS (\n INSERT INTO harness_runtime_connection\n (harness_id, id, replica_id, lease_expires_at)\n VALUES ($1, $2, $3, now() + make_interval(secs => $4))\n ON CONFLICT (harness_id, id) DO NOTHING\n )\n UPDATE harnesses SET last_connected_at = now() WHERE id = $1\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Uuid",
+ "Float8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "de149836f561243b3afb7c1b1b8f74e79f1a89a7097a81754ceb38ba85008990"
+}
diff --git a/.sqlx/query-e40e811009a56836ce10a1cd8d731a74aaaf7f7ddda8ff82abb89d6c10fc8524.json b/.sqlx/query-e40e811009a56836ce10a1cd8d731a74aaaf7f7ddda8ff82abb89d6c10fc8524.json
new file mode 100644
index 0000000000..1dc9029ec2
--- /dev/null
+++ b/.sqlx/query-e40e811009a56836ce10a1cd8d731a74aaaf7f7ddda8ff82abb89d6c10fc8524.json
@@ -0,0 +1,37 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO harness_runtime_connection\n (harness_id, id, replica_id, lease_expires_at)\n SELECT live.harness_id, live.id, $3, now() + make_interval(secs => $4)\n FROM UNNEST($1::uuid[], $2::uuid[]) AS live(harness_id, id)\n JOIN harnesses ON harnesses.id = live.harness_id\n ON CONFLICT (harness_id, id) DO UPDATE\n SET lease_expires_at = EXCLUDED.lease_expires_at\n RETURNING harness_id, id, (xmax = 0) AS \"restored!\"\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "harness_id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 2,
+ "name": "restored!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "UuidArray",
+ "Uuid",
+ "Float8"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ null
+ ]
+ },
+ "hash": "e40e811009a56836ce10a1cd8d731a74aaaf7f7ddda8ff82abb89d6c10fc8524"
+}
diff --git a/.sqlx/query-f1bfc93ff556e8fc22dbc7788dc5474aa042c379d0f255faa0b487c5480bffcb.json b/.sqlx/query-f1bfc93ff556e8fc22dbc7788dc5474aa042c379d0f255faa0b487c5480bffcb.json
deleted file mode 100644
index 9512791b63..0000000000
--- a/.sqlx/query-f1bfc93ff556e8fc22dbc7788dc5474aa042c379d0f255faa0b487c5480bffcb.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "UPDATE harnesses SET last_connected_at = now() WHERE id = $1",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid"
- ]
- },
- "nullable": []
- },
- "hash": "f1bfc93ff556e8fc22dbc7788dc5474aa042c379d0f255faa0b487c5480bffcb"
-}
diff --git a/crates/agent_harness/AGENTS.md b/crates/agent_harness/AGENTS.md
index cb16e7000d..3dd0ef8e60 100644
--- a/crates/agent_harness/AGENTS.md
+++ b/crates/agent_harness/AGENTS.md
@@ -94,6 +94,16 @@ Reconcile active leases against terminal log evidence too: an observer can miss
completion during takeover. Previously bound quiet attempts reconcile before new
wakes; interrupted quiet batches return to the next real prompt.
+Harness presence is per connection, because a runtime's redial can land on a
+different replica than the socket it replaced, and a stalled replica can notice
+that socket's end long after. `RuntimeRegistry` mints a `RuntimeConnectionId` per
+dial and reports it to `HarnessPresence`; a close removes only its own row, and
+each replica renews the leases of what it holds, so a crash reads offline within
+the lease. To debug a harness in Datadog, search `@harness:<id>` for
+`@telemetry_event:(harness_runtime_connected OR harness_runtime_disconnected OR
+harness_runtime_lease_restored)`: each carries `connection` and `replica`, and a
+disconnect's `other_live_connections` above 0 is a stale close that changed nothing.
+
## Checks
- `cargo nextest run -p agent_harness --lib --test-threads=2` (`SQLX_OFFLINE`
diff --git a/crates/agent_harness/src/domain/ports.rs b/crates/agent_harness/src/domain/ports.rs
index 48cdad2230..6145e06d37 100644
--- a/crates/agent_harness/src/domain/ports.rs
+++ b/crates/agent_harness/src/domain/ports.rs
@@ -143,20 +143,58 @@ pub trait CodingAgentSource: Send + Sync + 'static {
) -> impl Future<Output = anyhow::Result<Option<bool>>> + Send;
}
+/// One dial of a harness's runtime; a redial is a new connection.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct RuntimeConnectionId(macro_uuid::Uuid);
+
+impl RuntimeConnectionId {
+ /// Mint the identity of a new dial.
+ #[must_use]
+ pub fn generate() -> Self {
+ Self(macro_uuid::generate_uuid_v7())
+ }
+
+ /// The raw uuid, for persistence.
+ #[must_use]
+ pub fn as_uuid(self) -> macro_uuid::Uuid {
+ self.0
+ }
+}
+
+impl std::fmt::Display for RuntimeConnectionId {
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ self.0.fmt(formatter)
+ }
+}
+
/// Durable attach/detach bookkeeping for harness runtime connections.
///
/// The registry itself is in-process liveness; this is what lets the rest of
-/// the product (the harness settings page) see whether a daemon is up.
+/// the product (the harness settings page) see whether a daemon is up. Each
+/// report names its connection because a harness's redial can land on another
+/// replica, which then learns nothing about the older socket being closed here.
/// Methods take `Arc<Self>` and return owned futures so the registry can fire
/// them from its own background tasks.
pub trait HarnessPresence: Send + Sync + 'static {
/// A runtime attached for this harness.
- fn connected(self: Arc<Self>, harness: HarnessId) -> Pin<Box<dyn Future<Output = ()> + Send>>;
+ fn connected(
+ self: Arc<Self>,
+ harness: HarnessId,
+ connection: RuntimeConnectionId,
+ ) -> Pin<Box<dyn Future<Output = ()> + Send>>;
- /// This harness's runtime connection closed.
+ /// This connection closed. Other connections of the harness are unaffected.
fn disconnected(
self: Arc<Self>,
harness: HarnessId,
+ connection: RuntimeConnectionId,
+ ) -> Pin<Box<dyn Future<Output = ()> + Send>>;
+
+ /// These connections are still held by this process, so a crash of this
+ /// process stops vouching for them.
+ fn renew(
+ self: Arc<Self>,
+ live: Vec<(HarnessId, RuntimeConnectionId)>,
) -> Pin<Box<dyn Future<Output = ()> + Send>>;
}
diff --git a/crates/agent_harness/src/outbound/runtime_registry.rs b/crates/agent_harness/src/outbound/runtime_registry.rs
index 9025abb4d5..527e021049 100644
--- a/crates/agent_harness/src/outbound/runtime_registry.rs
+++ b/crates/agent_harness/src/outbound/runtime_registry.rs
@@ -22,8 +22,11 @@ use bot_id::BotId;
use dashmap::DashMap;
use harness_id::HarnessId;
use tokio::sync::mpsc;
+use tokio::task::JoinHandle;
-use crate::domain::ports::{HarnessBindings, HarnessPresence, RuntimeConnections};
+use crate::domain::ports::{
+ HarnessBindings, HarnessPresence, RuntimeConnectionId, RuntimeConnections,
+};
#[cfg(test)]
mod test;
@@ -42,9 +45,25 @@ pub struct LiveRuntimeSessions {
pub span: tracing::Span,
}
+/// A connection and the identity its presence reports carry.
+struct ListedConnection<Sender> {
+ id: RuntimeConnectionId,
+ connection: Arc<RuntimeConnection<Sender>>,
+}
+
+// Manual Clone impl: the sender itself is behind the Arc.
+impl<Sender> Clone for ListedConnection<Sender> {
+ fn clone(&self) -> Self {
+ Self {
+ id: self.id,
+ connection: Arc::clone(&self.connection),
+ }
+ }
+}
+
/// Every harness's live runtime connection.
pub struct RuntimeRegistry<Sender> {
- connections: DashMap<HarnessId, Arc<RuntimeConnection<Sender>>>,
+ connections: DashMap<HarnessId, ListedConnection<Sender>>,
presence: Option<Arc<dyn HarnessPresence>>,
live_sessions: Option<mpsc::UnboundedSender<LiveRuntimeSessions>>,
}
@@ -117,7 +136,7 @@ where
let connection = self
.connections
.get(&harness)
- .map(|entry| Arc::clone(&entry))?;
+ .map(|entry| Arc::clone(&entry.connection))?;
Some(connection.probe_models().await)
}
}
@@ -136,26 +155,51 @@ where
where
Carrier: Transport<ToRuntimeMessage, ToServerMessage, Sender = Sender>,
{
- let connection = RuntimeConnection::connect(carrier);
+ let listed = ListedConnection {
+ id: RuntimeConnectionId::generate(),
+ connection: RuntimeConnection::connect(carrier),
+ };
// Listed before it is watched: a socket that dies immediately would
// otherwise have its eviction run before the entry it is meant to
// remove exists, leaving the dead connection listed for good.
- if let Some(displaced) = self.connections.insert(harness, Arc::clone(&connection)) {
- displaced.evict();
- tracing::info!(%harness, "a redialed runtime replaced this harness's connection");
- }
- if let Some(presence) = &self.presence {
- tokio::spawn(Arc::clone(presence).connected(harness));
+ if let Some(displaced) = self.connections.insert(harness, listed.clone()) {
+ displaced.connection.evict();
+ tracing::info!(
+ %harness,
+ connection = %listed.id,
+ displaced_connection = %displaced.id,
+ "a redialed runtime replaced this harness's connection"
+ );
}
+ let connected_write = self
+ .presence
+ .as_ref()
+ .map(|presence| tokio::spawn(Arc::clone(presence).connected(harness, listed.id)));
if let Some(live_sessions) = &self.live_sessions {
tokio::spawn(report_live_sessions(
harness,
- Arc::clone(&connection),
+ Arc::clone(&listed.connection),
live_sessions.clone(),
));
}
- self.watch_for_close(harness, connection);
+ self.watch_for_close(harness, listed, connected_write);
+ }
+
+ /// Tell presence which connections this process still holds.
+ pub async fn renew_presence(&self) {
+ let Some(presence) = &self.presence else {
+ return;
+ };
+
+ let live: Vec<_> = self
+ .connections
+ .iter()
+ .map(|entry| (*entry.key(), entry.id))
+ .collect();
+ if !live.is_empty() {
+ Arc::clone(presence).renew(live).await;
+ }
}
/// Drop `connection` from the registry once its transport ends.
@@ -168,9 +212,10 @@ where
fn watch_for_close(
self: &Arc<Self>,
harness: HarnessId,
- connection: Arc<RuntimeConnection<Sender>>,
+ listed: ListedConnection<Sender>,
+ connected_write: Option<JoinHandle<()>>,
) {
- tokio::spawn(Arc::clone(self).drop_when_closed(harness, connection));
+ tokio::spawn(Arc::clone(self).drop_when_closed(harness, listed, connected_write));
}
/// The body of that watch: wait for the end, then unlist it.
@@ -180,23 +225,31 @@ where
async fn drop_when_closed(
self: Arc<Self>,
harness: HarnessId,
- connection: Arc<RuntimeConnection<Sender>>,
+ listed: ListedConnection<Sender>,
+ connected_write: Option<JoinHandle<()>>,
) {
- connection.closed().await;
+ listed.connection.closed().await;
// Only if it is still this connection: a redial that already replaced
// it owns the entry now, and evicting that would take the live socket
- // down along with the dead one. The presence write follows the same
- // rule, so a displaced socket's death never marks the live redial
- // disconnected.
+ // down along with the dead one.
let removed = self
.connections
- .remove_if(&harness, |_, current| Arc::ptr_eq(current, &connection))
+ .remove_if(&harness, |_, current| current.id == listed.id)
.is_some();
- if removed {
- if let Some(presence) = &self.presence {
- tokio::spawn(Arc::clone(presence).disconnected(harness));
+ if let Some(presence) = &self.presence {
+ // A socket that dies immediately must not have its close recorded
+ // before its attach, which would leave the attach standing.
+ if let Some(connected_write) = connected_write {
+ let _ = connected_write.await;
}
- tracing::info!(%harness, "dropped this harness's closed runtime connection");
+ Arc::clone(presence).disconnected(harness, listed.id).await;
+ }
+ if removed {
+ tracing::info!(
+ %harness,
+ connection = %listed.id,
+ "dropped this harness's closed runtime connection"
+ );
}
}
}
@@ -281,7 +334,7 @@ where
.registry
.connections
.get(&harness)
- .map(|entry| Arc::clone(&entry))?;
+ .map(|entry| Arc::clone(&entry.connection))?;
Some(connection.bind(session).await)
}
diff --git a/crates/agent_harness/src/outbound/runtime_registry/test.rs b/crates/agent_harness/src/outbound/runtime_registry/test.rs
index 27978644f9..f5df5702c2 100644
--- a/crates/agent_harness/src/outbound/runtime_registry/test.rs
+++ b/crates/agent_harness/src/outbound/runtime_registry/test.rs
@@ -164,7 +164,7 @@ async fn a_closed_connection_is_unlisted_so_nothing_binds_onto_it() {
let connection = registry
.connections
.get(&HarnessId::TEST_A)
- .map(|entry| Arc::clone(&entry))
+ .map(|entry| entry.value().clone())
.expect("the dial is listed");
let bound = connections(
Arc::new(FakeBindings::bound(BotId::TEST_A, HarnessId::TEST_A)),
@@ -174,7 +174,7 @@ async fn a_closed_connection_is_unlisted_so_nothing_binds_onto_it() {
runtime.disconnects();
// The same work `attach` spawns, awaited here rather than raced.
Arc::clone(&registry)
- .drop_when_closed(HarnessId::TEST_A, connection)
+ .drop_when_closed(HarnessId::TEST_A, connection, None)
.await;
assert!(!registry.is_connected(HarnessId::TEST_A));
@@ -194,7 +194,7 @@ async fn a_redials_connection_outlives_the_one_it_displaced() {
let displaced = registry
.connections
.get(&HarnessId::TEST_A)
- .map(|entry| Arc::clone(&entry))
+ .map(|entry| entry.value().clone())
.expect("the first dial is listed");
let bound = connections(
Arc::new(FakeBindings::bound(BotId::TEST_A, HarnessId::TEST_A)),
@@ -204,7 +204,7 @@ async fn a_redials_connection_outlives_the_one_it_displaced() {
registry.attach(HarnessId::TEST_A, ContainerMock::default());
first.disconnects();
Arc::clone(&registry)
- .drop_when_closed(HarnessId::TEST_A, displaced)
+ .drop_when_closed(HarnessId::TEST_A, displaced, None)
.await;
// The dead connection's eviction must not unlist the live one that took
@@ -218,61 +218,167 @@ async fn a_redials_connection_outlives_the_one_it_displaced() {
);
}
-#[tokio::test]
-async fn attach_and_close_report_presence() {
- #[derive(Default)]
- struct RecordingPresence {
- events: Mutex<Vec<(HarnessId, &'static str)>>,
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum PresenceEvent {
+ Connected(RuntimeConnectionId),
+ Disconnected(RuntimeConnectionId),
+}
+
+#[derive(Default)]
+struct RecordingPresence {
+ events: Mutex<Vec<PresenceEvent>>,
+ renewed: Mutex<Vec<Vec<(HarnessId, RuntimeConnectionId)>>>,
+}
+
+impl HarnessPresence for RecordingPresence {
+ fn connected(
+ self: Arc<Self>,
+ _harness: HarnessId,
+ connection: RuntimeConnectionId,
+ ) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>> {
+ Box::pin(async move {
+ self.events
+ .lock()
+ .unwrap()
+ .push(PresenceEvent::Connected(connection));
+ })
}
- impl HarnessPresence for RecordingPresence {
- fn connected(
- self: Arc<Self>,
- harness: HarnessId,
- ) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>> {
- Box::pin(async move {
- self.events.lock().unwrap().push((harness, "connected"));
- })
- }
+ fn disconnected(
+ self: Arc<Self>,
+ _harness: HarnessId,
+ connection: RuntimeConnectionId,
+ ) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>> {
+ Box::pin(async move {
+ self.events
+ .lock()
+ .unwrap()
+ .push(PresenceEvent::Disconnected(connection));
+ })
+ }
- fn disconnected(
- self: Arc<Self>,
- harness: HarnessId,
- ) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>> {
- Box::pin(async move {
- self.events.lock().unwrap().push((harness, "disconnected"));
- })
- }
+ fn renew(
+ self: Arc<Self>,
+ live: Vec<(HarnessId, RuntimeConnectionId)>,
+ ) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>> {
+ Box::pin(async move {
+ self.renewed.lock().unwrap().push(live);
+ })
}
+}
- let presence = Arc::new(RecordingPresence::default());
- let registry: Arc<RuntimeRegistry<ContainerSender>> = RuntimeRegistry::with_presence(
- Arc::clone(&presence) as Arc<dyn HarnessPresence>,
+fn registry_with_presence(
+ presence: &Arc<RecordingPresence>,
+) -> Arc<RuntimeRegistry<ContainerSender>> {
+ RuntimeRegistry::with_presence(
+ Arc::clone(presence) as Arc<dyn HarnessPresence>,
mpsc::unbounded_channel().0,
- );
- let runtime = ContainerMock::default();
- registry.attach(HarnessId::TEST_A, runtime.clone());
- let connection = registry
+ )
+}
+
+fn listed_connection(
+ registry: &Arc<RuntimeRegistry<ContainerSender>>,
+) -> ListedConnection<ContainerSender> {
+ registry
.connections
.get(&HarnessId::TEST_A)
- .map(|entry| Arc::clone(&entry))
- .expect("the dial is listed");
+ .map(|entry| entry.value().clone())
+ .expect("the dial is listed")
+}
- runtime.disconnects();
+#[tokio::test]
+async fn a_displaced_connections_close_reports_only_that_connection() {
+ let presence = Arc::new(RecordingPresence::default());
+ let registry = registry_with_presence(&presence);
+ let first = ContainerMock::default();
+ registry.attach(HarnessId::TEST_A, first.clone());
+ let displaced = listed_connection(&registry);
+ registry.attach(HarnessId::TEST_A, ContainerMock::default());
+ let current = listed_connection(&registry);
+
+ first.disconnects();
Arc::clone(&registry)
- .drop_when_closed(HarnessId::TEST_A, connection)
+ .drop_when_closed(HarnessId::TEST_A, displaced.clone(), None)
.await;
- // The connected write is spawned; give it a beat to land.
- for _ in 0..100 {
- if presence.events.lock().unwrap().len() >= 2 {
- break;
+ // Presence is told about the dead connection, by name, so its write cannot
+ // be mistaken for the live connection going away.
+ let events = presence.events.lock().unwrap().clone();
+ assert!(events.contains(&PresenceEvent::Disconnected(displaced.id)));
+ assert!(!events.contains(&PresenceEvent::Disconnected(current.id)));
+ assert_eq!(listed_connection(&registry).id, current.id);
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_close_waits_for_its_own_connected_write() {
+ let presence = Arc::new(RecordingPresence::default());
+ let registry = registry_with_presence(&presence);
+ let runtime = ContainerMock::default();
+ // Not through `attach`, whose own watch would report the close unprompted.
+ let listed = ListedConnection {
+ id: RuntimeConnectionId::generate(),
+ connection: RuntimeConnection::connect(runtime.clone()),
+ };
+ registry
+ .connections
+ .insert(HarnessId::TEST_A, listed.clone());
+ let (release, released) = tokio::sync::oneshot::channel::<()>();
+ let connected_write = tokio::spawn({
+ let presence = Arc::clone(&presence);
+ let id = listed.id;
+ async move {
+ let _ = released.await;
+ presence
+ .events
+ .lock()
+ .unwrap()
+ .push(PresenceEvent::Connected(id));
}
- tokio::task::yield_now().await;
+ });
+ runtime.disconnects();
+
+ let closing = Arc::clone(&registry).drop_when_closed(
+ HarnessId::TEST_A,
+ listed.clone(),
+ Some(connected_write),
+ );
+ tokio::pin!(closing);
+ tokio::select! {
+ () = &mut closing => panic!("the close finished before its attach was recorded"),
+ () = tokio::time::sleep(Duration::from_secs(1)) => {}
}
- let events = presence.events.lock().unwrap().clone();
- assert!(events.contains(&(HarnessId::TEST_A, "connected")));
- assert!(events.contains(&(HarnessId::TEST_A, "disconnected")));
+ // The registry already stopped listing it; only the presence write waited.
+ assert!(!registry.is_connected(HarnessId::TEST_A));
+ assert!(presence.events.lock().unwrap().is_empty());
+
+ release.send(()).expect("the connected write is waiting");
+ closing.await;
+
+ assert_eq!(
+ *presence.events.lock().unwrap(),
+ [
+ PresenceEvent::Connected(listed.id),
+ PresenceEvent::Disconnected(listed.id)
+ ]
+ );
+}
+
+#[tokio::test]
+async fn renewal_lists_the_connections_this_process_holds() {
+ let presence = Arc::new(RecordingPresence::default());
+ let registry = registry_with_presence(&presence);
+
+ registry.renew_presence().await;
+ assert!(presence.renewed.lock().unwrap().is_empty());
+
+ registry.attach(HarnessId::TEST_A, ContainerMock::default());
+ let listed = listed_connection(&registry);
+ registry.renew_presence().await;
+
+ assert_eq!(
+ *presence.renewed.lock().unwrap(),
+ [vec![(HarnessId::TEST_A, listed.id)]]
+ );
}
#[tokio::test]
diff --git a/crates/coding_agent_worker/AGENTS.md b/crates/coding_agent_worker/AGENTS.md
index ec8526a1b4..9302df8910 100644
--- a/crates/coding_agent_worker/AGENTS.md
+++ b/crates/coding_agent_worker/AGENTS.md
@@ -26,6 +26,11 @@ the local ACP process (Claude Code, Codex, ...), and drives native
what the session has. General Macro guidance stays in the `SelfKnowledge` tool.
- `src/trigger.rs` + `src/dispatch.rs`: agent-trigger broker events → session
create/dial/prompt against the live harness service.
+- `src/daemon.rs` `watch_presence`: each minute a connection has held, asks
+ `GET /harnesses/me` with `RUNTIME_CONNECTED_HEADER`. A service that reads the
+ harness offline logs `harness_presence_disagreement`, macrod warns, and the TUI
+ header says "runtime connected, service reports offline" instead of "paired,
+ runtime not dialed in".
- `src/tui.rs` (`src/tui/`): the control panel itself (registration, agents,
sessions, config form, logs).
- `src/herdr.rs` (`src/herdr/`): native herdr integration — only compiled
diff --git a/crates/coding_agent_worker/src/daemon.rs b/crates/coding_agent_worker/src/daemon.rs
index acbf9c2c9c..460f0ad741 100644
--- a/crates/coding_agent_worker/src/daemon.rs
+++ b/crates/coding_agent_worker/src/daemon.rs
@@ -7,7 +7,9 @@
use std::path::Path;
use std::time::Duration;
+use tokio::sync::watch;
use tokio_util::sync::CancellationToken;
+use tokio_util::task::AbortOnDropHandle;
use crate::config::{Config, HarnessCredentials};
use crate::dispatch::Dispatcher;
@@ -28,6 +30,9 @@ mod test;
/// worst-case window where a brand-new agent is deaf.
const BOUND_AGENT_REFRESH: Duration = Duration::from_secs(10);
+/// How often a held runtime connection is compared with what the service says.
+const PRESENCE_CHECK_EVERY: Duration = Duration::from_secs(60);
+
/// A running serving core: SSE listener, harness bridge.
pub struct Daemon {
cancel: CancellationToken,
@@ -38,6 +43,8 @@ pub struct Daemon {
expect(dead_code, reason = "agents are handed off only on Unix")
)]
handoff: HandoffSender,
+ runtime_connected: watch::Receiver<bool>,
+ _presence_watch: AbortOnDropHandle<()>,
}
/// The agent a previous macrod process image handed over, ready to relay.
@@ -102,6 +109,11 @@ impl Daemon {
adopted,
handoffs,
);
+ let runtime_connected = runtime.connection_state();
+ let presence_watch = AbortOnDropHandle::new(tokio::spawn(watch_presence(
+ EventStreamClient::new(&config.macro_api, &credentials),
+ runtime_connected.clone(),
+ )));
let executor = Dispatcher::new(api, runtime, config.workspace.clone());
#[cfg(unix)]
let executor = executor.with_workspaces(workspaces);
@@ -150,9 +162,16 @@ impl Daemon {
task,
turns,
handoff,
+ runtime_connected,
+ _presence_watch: presence_watch,
})
}
+ /// Whether this daemon currently holds a runtime connection.
+ pub fn runtime_connected(&self) -> bool {
+ *self.runtime_connected.borrow()
+ }
+
/// Whether the serving task is still alive.
pub fn is_running(&self) -> bool {
!self.task.is_finished()
@@ -195,6 +214,47 @@ impl Daemon {
}
}
+/// Compare a held runtime connection with what the service reports.
+///
+/// The service reading this harness as offline makes the app treat its agents as
+/// unavailable, and nothing else on this machine would notice: the socket is
+/// healthy. Warns once per disagreement; the check itself lets the service log
+/// each one for its monitor.
+async fn watch_presence(client: EventStreamClient, mut runtime: watch::Receiver<bool>) {
+ let mut ticker = tokio::time::interval(PRESENCE_CHECK_EVERY);
+ ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+ let mut disagreeing = false;
+ loop {
+ ticker.tick().await;
+
+ if !runtime_held_since_last_look(&mut runtime) {
+ disagreeing = false;
+ continue;
+ }
+
+ match client.service_reports_connected().await {
+ Ok(false) if !disagreeing => {
+ disagreeing = true;
+ tracing::warn!("macrod holds a runtime connection the service reports as offline");
+ }
+ Ok(true) if disagreeing => {
+ disagreeing = false;
+ tracing::info!("the service reports this harness connected again");
+ }
+ Ok(_) => {}
+ Err(error) => tracing::debug!(error = %error, "could not check harness presence"),
+ }
+ }
+}
+
+/// Whether the runtime has stayed connected since this was last asked. A fresh
+/// dial does not count yet: the service's record of it may still be landing.
+fn runtime_held_since_last_look(runtime: &mut watch::Receiver<bool>) -> bool {
+ let redialed = runtime.has_changed().unwrap_or(true);
+ let connected = *runtime.borrow_and_update();
+ connected && !redialed
+}
+
/// Relay an inherited agent instead of spawning one, continuing its turns.
#[cfg(unix)]
fn adopt(inherited: Option<InheritedAgent>, turns: &ActiveTurns) -> Option<RunningAgent> {
diff --git a/crates/coding_agent_worker/src/daemon/test.rs b/crates/coding_agent_worker/src/daemon/test.rs
index 2da8815a4d..ca47cb97be 100644
--- a/crates/coding_agent_worker/src/daemon/test.rs
+++ b/crates/coding_agent_worker/src/daemon/test.rs
@@ -1,5 +1,24 @@
use super::*;
+#[test]
+fn a_fresh_dial_is_not_a_held_connection_until_it_has_lasted() {
+ let (connected, mut runtime) = watch::channel(false);
+ assert!(!runtime_held_since_last_look(&mut runtime));
+
+ connected.send_replace(true);
+ assert!(!runtime_held_since_last_look(&mut runtime));
+ assert!(runtime_held_since_last_look(&mut runtime));
+
+ // A drop and redial between looks starts the wait over.
+ connected.send_replace(false);
+ connected.send_replace(true);
+ assert!(!runtime_held_since_last_look(&mut runtime));
+ assert!(runtime_held_since_last_look(&mut runtime));
+
+ connected.send_replace(false);
+ assert!(!runtime_held_since_last_look(&mut runtime));
+}
+
#[tokio::test]
async fn stop_does_not_wait_for_in_flight_work() {
let daemon = Daemon {
@@ -7,6 +26,8 @@ async fn stop_does_not_wait_for_in_flight_work() {
task: tokio::spawn(std::future::pending()),
turns: ActiveTurns::default(),
handoff: crate::harness::Handoffs::channel().1,
+ runtime_connected: watch::channel(false).1,
+ _presence_watch: AbortOnDropHandle::new(tokio::spawn(async {})),
};
tokio::time::timeout(Duration::from_secs(1), daemon.stop())
diff --git a/crates/coding_agent_worker/src/outbound/stream.rs b/crates/coding_agent_worker/src/outbound/stream.rs
index f3027a8bc2..7e8ff469e6 100644
--- a/crates/coding_agent_worker/src/outbound/stream.rs
+++ b/crates/coding_agent_worker/src/outbound/stream.rs
@@ -10,10 +10,11 @@
use std::collections::BTreeSet;
use std::marker::PhantomData;
use std::pin::Pin;
+use std::time::Duration;
use eventsource_stream::{Event, EventStreamError, Eventsource as _};
use futures::{Stream, StreamExt as _};
-use harnesses::domain::models::{Harness, HarnessAgent, HarnessOwner};
+use harnesses::domain::models::{Harness, HarnessAgent, HarnessOwner, RUNTIME_CONNECTED_HEADER};
use rootcause::prelude::ResultExt as _;
use serde::de::DeserializeOwned;
use webhook::domain::models::{WebhookFilter, WebhookScope};
@@ -25,6 +26,10 @@ mod test;
const HARNESS_TOKEN_HEADER: &str = "x-macro-harness-token";
+/// Set on the presence request alone: a timeout on the client would also end the
+/// long-lived event stream.
+const PRESENCE_CHECK_TIMEOUT: Duration = Duration::from_secs(10);
+
/// Map the harness's ownership onto the stream query's workspace scope.
pub fn stream_scope(owner: &HarnessOwner) -> WebhookScope {
match owner {
@@ -94,6 +99,21 @@ impl EventStreamClient {
Ok(stream_scope(&harness.owner))
}
+ /// Whether the service reports this harness connected, telling it that this
+ /// daemon holds a runtime connection so it can log any disagreement.
+ pub async fn service_reports_connected(&self) -> rootcause::Result<bool> {
+ let harness: Harness = self
+ .read(
+ "check this harness's presence",
+ self.http
+ .get(format!("{}/harnesses/me", self.base))
+ .header(RUNTIME_CONNECTED_HEADER, "true")
+ .timeout(PRESENCE_CHECK_TIMEOUT),
+ )
+ .await?;
+ Ok(harness.connected)
+ }
+
/// The agents currently bound to this harness, as sorted bot-id strings —
/// the shape the stream filter carries.
pub async fn bound_bot_ids(&self) -> rootcause::Result<Vec<String>> {
diff --git a/crates/coding_agent_worker/src/outbound/stream/test.rs b/crates/coding_agent_worker/src/outbound/stream/test.rs
index f35bff745e..01426c9a5f 100644
--- a/crates/coding_agent_worker/src/outbound/stream/test.rs
+++ b/crates/coding_agent_worker/src/outbound/stream/test.rs
@@ -29,6 +29,33 @@ struct Envelope {
n: u8,
}
+#[tokio::test]
+async fn presence_check_tells_the_service_a_connection_is_held() {
+ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
+
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("bind test listener");
+ let url = format!(
+ "http://{}",
+ listener.local_addr().expect("listener address")
+ );
+ let server = tokio::spawn(async move {
+ let (mut socket, _) = listener.accept().await.expect("accept");
+ let mut buf = vec![0u8; 4096];
+ let read = socket.read(&mut buf).await.unwrap_or(0);
+ let _ = socket.write_all(TEAM_HARNESS.as_bytes()).await;
+ let _ = socket.shutdown().await;
+ String::from_utf8_lossy(&buf[..read]).to_lowercase()
+ });
+
+ assert!(test_client(&url).service_reports_connected().await.unwrap());
+
+ let request = server.await.unwrap();
+ assert!(request.starts_with("get /harnesses/me "));
+ assert!(request.contains("x-macro-harness-runtime-connected: true"));
+}
+
#[tokio::test]
async fn connect_yields_typed_envelopes_and_skips_keep_alives_and_junk() {
let (url, requests) = serve(vec![
diff --git a/crates/coding_agent_worker/src/runtime.rs b/crates/coding_agent_worker/src/runtime.rs
index 596e01b6e3..c832c16b63 100644
--- a/crates/coding_agent_worker/src/runtime.rs
+++ b/crates/coding_agent_worker/src/runtime.rs
@@ -66,6 +66,11 @@ impl Runtime {
}
}
+ /// Follows whether the runtime connection is currently up.
+ pub fn connection_state(&self) -> watch::Receiver<bool> {
+ self.connected.clone()
+ }
+
/// Wait briefly for the supervisor to connect before dispatching a prompt.
/// Callers never open a second connection while it is reconnecting.
pub async fn ensure_connected(&self) -> Result<(), tungstenite::Error> {
diff --git a/crates/coding_agent_worker/src/tui/api.rs b/crates/coding_agent_worker/src/tui/api.rs
index fc1a4fa555..f72bcc4797 100644
--- a/crates/coding_agent_worker/src/tui/api.rs
+++ b/crates/coding_agent_worker/src/tui/api.rs
@@ -1,5 +1,7 @@
//! The TUI's read/act client against the harness APIs, as the harness.
+use std::time::Duration;
+
use harnesses::domain::models::{Harness, HarnessAgent, HarnessSession};
use reqwest::StatusCode;
use rootcause::prelude::ResultExt as _;
@@ -9,6 +11,10 @@ use crate::config::HarnessCredentials;
const HARNESS_TOKEN_HEADER: &str = "x-macro-harness-token";
+/// A dashboard read that takes longer than this is not worth waiting on: the
+/// next refresh asks again.
+const READ_TIMEOUT: Duration = Duration::from_secs(10);
+
/// What the server currently knows about this daemon.
#[derive(Debug, Clone, Default)]
pub struct Snapshot {
@@ -50,9 +56,15 @@ impl HarnessSelfApi {
.http
.get(format!("{}{path}", self.base))
.header(HARNESS_TOKEN_HEADER, &self.token)
+ .timeout(READ_TIMEOUT)
.send()
.await
- .context(format!("could not reach the service to {what}"))?;
+ .map_err(|error| {
+ rootcause::report!(
+ "could not reach the service to {what}: {}",
+ transport_failure(error)
+ )
+ })?;
// A dead credential (revoked harness) answers 401; surface that as
// "no harness" rather than an error so the UI offers re-pairing.
if response.status() == StatusCode::UNAUTHORIZED
@@ -114,3 +126,18 @@ impl HarnessSelfApi {
Ok(())
}
}
+
+/// A transport error with its causes on one line. reqwest's own message stops at
+/// "error sending request for url", leaving out the reason, and the URL it leads
+/// with would push the reason out of the one-line footer that shows this.
+fn transport_failure(error: reqwest::Error) -> String {
+ let error = error.without_url();
+ let mut failure = error.to_string();
+ let mut source = std::error::Error::source(&error);
+ while let Some(cause) = source {
+ failure.push_str(": ");
+ failure.push_str(&cause.to_string());
+ source = cause.source();
+ }
+ failure
+}
diff --git a/crates/coding_agent_worker/src/tui/app.rs b/crates/coding_agent_worker/src/tui/app.rs
index 2001573b96..d288be0edf 100644
--- a/crates/coding_agent_worker/src/tui/app.rs
+++ b/crates/coding_agent_worker/src/tui/app.rs
@@ -21,6 +21,7 @@ use crate::outbound::pairing::{ClaimStatus, PairingClient, machine_name};
/// How often the dashboard re-reads the server.
pub(crate) const REFRESH_EVERY: Duration = Duration::from_secs(3);
+const REFRESH_FAILED: &str = "refresh failed";
/// Which tab is showing.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -174,6 +175,11 @@ impl App {
}
/// (Re)start the serving core on the current config and credential.
+ /// Whether this process currently holds a runtime connection.
+ pub(crate) fn runtime_connected(&self) -> bool {
+ self.daemon.as_ref().is_some_and(Daemon::runtime_connected)
+ }
+
pub(crate) async fn restart_daemon(&mut self) -> bool {
if let Some(daemon) = self.daemon.take() {
daemon.stop().await;
@@ -250,11 +256,28 @@ impl App {
Ok(snapshot) => {
self.snapshot = snapshot;
self.last_refresh = Some(Instant::now());
+ // A failure this refresh recovered from is no longer news, and
+ // left up it reads as a live problem.
+ if self.showing_refresh_failure() {
+ self.status = None;
+ }
+ }
+ Err(error) => {
+ // Failed refreshes retry on every tick; log the first of a run.
+ if !self.showing_refresh_failure() {
+ tracing::warn!(error = %error, "could not refresh the dashboard");
+ }
+ self.fail(format!("{REFRESH_FAILED}: {error}"));
}
- Err(error) => self.fail(format!("refresh failed: {error}")),
}
}
+ fn showing_refresh_failure(&self) -> bool {
+ self.status
+ .as_ref()
+ .is_some_and(|(message, _)| message.starts_with(REFRESH_FAILED))
+ }
+
pub(crate) async fn maybe_refresh(&mut self) {
let due = self
.last_refresh
diff --git a/crates/coding_agent_worker/src/tui/ui/shell.rs b/crates/coding_agent_worker/src/tui/ui/shell.rs
index 0288f3676c..3d9e49c586 100644
--- a/crates/coding_agent_worker/src/tui/ui/shell.rs
+++ b/crates/coding_agent_worker/src/tui/ui/shell.rs
@@ -10,6 +10,11 @@ use crate::tui::app::{App, Mode, Tab};
pub(super) fn render_header(frame: &mut Frame, app: &App, area: Rect) {
let (dot, dot_style, state) = match &app.snapshot.harness {
Some(harness) if harness.connected => ("●", Style::new().fg(OK), "connected"),
+ Some(_) if app.runtime_connected() => (
+ "●",
+ Style::new().fg(ERR),
+ "runtime connected, service reports offline",
+ ),
Some(_) => ("●", Style::new().fg(WARN), "paired, runtime not dialed in"),
None if app.paired() => ("○", Style::new().fg(WARN), "paired, unreachable"),
None => ("○", Style::new().fg(DIM), "not paired"),
diff --git a/crates/harnesses/AGENTS.md b/crates/harnesses/AGENTS.md
index bca58b4bee..9b95a86f8b 100644
--- a/crates/harnesses/AGENTS.md
+++ b/crates/harnesses/AGENTS.md
@@ -39,6 +39,16 @@ daemon reconciles its webhook feed from.
- `cargo test -p harnesses` (`SQLX_OFFLINE` unset).
+## Gotchas
+
+- `Harness.connected` reads `harness_runtime_connection`: true while a row has an
+ unexpired lease. `agent_harness_service` writes those rows per connection. With
+ no rows at all it falls back to `last_connected_at > last_disconnected_at`, for
+ connections held by replicas older than the rows; remove that once none remain.
+- `GET /harnesses/me` logs `harness_presence_disagreement` when the daemon sends
+ `RUNTIME_CONNECTED_HEADER` and the harness reads offline
+ (`infra/stacks/monitoring/monitors/harness-presence.ts` alerts on it).
+
## Related
- Rules: `.agents/rules/rust/architecture.md`, `.agents/rules/rust/security.md`
diff --git a/crates/harnesses/src/domain/models.rs b/crates/harnesses/src/domain/models.rs
index 1f6553697f..41e73c25c8 100644
--- a/crates/harnesses/src/domain/models.rs
+++ b/crates/harnesses/src/domain/models.rs
@@ -55,6 +55,10 @@ pub struct Harness {
pub last_connected_at: Option<DateTime<Utc>>,
}
+/// Sent by a daemon reading its own harness while it holds a runtime connection,
+/// so the service can log when its own answer disagrees.
+pub const RUNTIME_CONNECTED_HEADER: &str = "x-macro-harness-runtime-connected";
+
/// Request to open a pairing: the daemon asks for a code the user approves.
///
/// The daemon serializes this, so both derives are used.
diff --git a/crates/harnesses/src/inbound/axum_router.rs b/crates/harnesses/src/inbound/axum_router.rs
index f88456f3ec..0e67e59e4c 100644
--- a/crates/harnesses/src/inbound/axum_router.rs
+++ b/crates/harnesses/src/inbound/axum_router.rs
@@ -5,7 +5,7 @@ use std::sync::Arc;
use axum::{
Json, Router,
extract::{FromRef, Path, State},
- http::StatusCode,
+ http::{HeaderMap, StatusCode},
response::{IntoResponse, Response},
routing::{delete, get, patch, post},
};
@@ -22,11 +22,14 @@ use crate::domain::{
models::{
ApprovePairingRequest, ClaimOutcome, ClaimPairingRequest, ClaimedPairing,
CreatePairingRequest, CreatedPairing, Harness, HarnessAgent, HarnessSession,
- PairingDetails, PatchHarnessRequest,
+ PairingDetails, PatchHarnessRequest, RUNTIME_CONNECTED_HEADER,
},
ports::{HarnessError, HarnessService},
};
+#[cfg(test)]
+mod test;
+
/// State for the harnesses router.
pub struct HarnessesRouterState<S, Auth> {
service: Arc<S>,
@@ -352,13 +355,29 @@ pub async fn delete_harness_handler<S: HarnessService, Auth: MacroAuthorizationS
pub async fn get_self_harness_handler<S: HarnessService, Auth: MacroAuthorizationService>(
State(state): State<HarnessesRouterState<S, Auth>>,
authorization: MacroAuthorizationExtractor<Auth, HarnessOnly>,
+ headers: HeaderMap,
) -> Result<Json<Harness>, HarnessesHandlerErr> {
- Ok(Json(
- state
- .service
- .get_self(authorization.authorization.harness_id)
- .await?,
- ))
+ let harness = state
+ .service
+ .get_self(authorization.authorization.harness_id)
+ .await?;
+
+ if runtime_presence_disagrees(&headers, &harness) {
+ tracing::warn!(
+ telemetry_event = "harness_presence_disagreement",
+ harness = %harness.id,
+ last_connected_at = ?harness.last_connected_at,
+ "a daemon holds a runtime connection this harness reads as offline"
+ );
+ }
+
+ Ok(Json(harness))
+}
+
+/// Whether the daemon asking says it holds a connection the harness reads as not
+/// having. Only the daemon can see this: the service's own answer is what is wrong.
+fn runtime_presence_disagrees(headers: &HeaderMap, harness: &Harness) -> bool {
+ headers.contains_key(RUNTIME_CONNECTED_HEADER) && !harness.connected
}
/// Handler for `DELETE /harnesses/me`.
diff --git a/crates/harnesses/src/inbound/axum_router/test.rs b/crates/harnesses/src/inbound/axum_router/test.rs
new file mode 100644
index 0000000000..adc25c81bf
--- /dev/null
+++ b/crates/harnesses/src/inbound/axum_router/test.rs
@@ -0,0 +1,35 @@
+use axum::http::HeaderValue;
+use chrono::Utc;
+
+use super::*;
+use crate::domain::models::HarnessOwner;
+
+fn harness(connected: bool) -> Harness {
+ Harness {
+ id: HarnessId::TEST_A,
+ allow_permission_bypass: false,
+ kind: "macrod".to_owned(),
+ name: "desk".to_owned(),
+ owner: HarnessOwner::User {
+ user_id: "macro|[email protected]".to_owned(),
+ },
+ created_by: "macro|[email protected]".to_owned(),
+ created_at: Utc::now(),
+ updated_at: Utc::now(),
+ connected,
+ last_connected_at: None,
+ }
+}
+
+#[test]
+fn only_a_daemon_holding_a_connection_can_disagree_with_offline() {
+ let mut holding = HeaderMap::new();
+ holding.insert(RUNTIME_CONNECTED_HEADER, HeaderValue::from_static("true"));
+
+ assert!(runtime_presence_disagrees(&holding, &harness(false)));
+ assert!(!runtime_presence_disagrees(&holding, &harness(true)));
+ assert!(!runtime_presence_disagrees(
+ &HeaderMap::new(),
+ &harness(false)
+ ));
+}
diff --git a/crates/harnesses/src/outbound/pg_harness_repo.rs b/crates/harnesses/src/outbound/pg_harness_repo.rs
index daa9eccbc1..d7ced98ddd 100644
--- a/crates/harnesses/src/outbound/pg_harness_repo.rs
+++ b/crates/harnesses/src/outbound/pg_harness_repo.rs
@@ -44,22 +44,41 @@ struct HarnessRow {
updated_at: DateTime<Utc>,
last_connected_at: Option<DateTime<Utc>>,
last_disconnected_at: Option<DateTime<Utc>>,
+ has_live_connection: bool,
+ has_connection_record: bool,
+}
+
+impl HarnessRow {
+ /// A live connection row is the answer. A row whose lease lapsed means the
+ /// replica holding it died. With no rows at all, the connection may come
+ /// from a replica older than these rows, which only stamps the timestamps.
+ fn connected(&self) -> bool {
+ if self.has_live_connection {
+ return true;
+ }
+
+ if self.has_connection_record {
+ return false;
+ }
+
+ match (self.last_connected_at, self.last_disconnected_at) {
+ (Some(connected_at), Some(disconnected_at)) => connected_at > disconnected_at,
+ (Some(_), None) => true,
+ (None, _) => false,
+ }
+ }
}
impl TryFrom<HarnessRow> for Harness {
type Error = anyhow::Error;
fn try_from(row: HarnessRow) -> Result<Self, Self::Error> {
+ let connected = row.connected();
let owner = match (row.owner_user_id, row.team_id) {
(Some(user_id), None) => HarnessOwner::User { user_id },
(None, Some(team_id)) => HarnessOwner::Team { team_id },
_ => anyhow::bail!("harness row violated its owner invariant"),
};
- let connected = match (row.last_connected_at, row.last_disconnected_at) {
- (Some(connected_at), Some(disconnected_at)) => connected_at > disconnected_at,
- (Some(_), None) => true,
- (None, _) => false,
- };
Ok(Self {
id: HarnessId::new_from_uuid(row.id),
@@ -94,7 +113,15 @@ async fn fetch_harness(
r#"
SELECT
id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,
- created_at, updated_at, last_connected_at, last_disconnected_at
+ created_at, updated_at, last_connected_at, last_disconnected_at,
+ EXISTS (
+ SELECT 1 FROM harness_runtime_connection c
+ WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()
+ ) AS "has_live_connection!",
+ EXISTS (
+ SELECT 1 FROM harness_runtime_connection c
+ WHERE c.harness_id = harnesses.id
+ ) AS "has_connection_record!"
FROM harnesses
WHERE id = $1 AND deleted_at IS NULL
"#,
@@ -338,7 +365,15 @@ impl HarnessRepo for PgHarnessRepo {
r#"
SELECT
id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,
- created_at, updated_at, last_connected_at, last_disconnected_at
+ created_at, updated_at, last_connected_at, last_disconnected_at,
+ EXISTS (
+ SELECT 1 FROM harness_runtime_connection c
+ WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()
+ ) AS "has_live_connection!",
+ EXISTS (
+ SELECT 1 FROM harness_runtime_connection c
+ WHERE c.harness_id = harnesses.id
+ ) AS "has_connection_record!"
FROM harnesses
WHERE deleted_at IS NULL
AND (
@@ -376,7 +411,15 @@ impl HarnessRepo for PgHarnessRepo {
WHERE id = $1 AND deleted_at IS NULL
RETURNING
id, kind, name, owner_user_id, team_id, created_by, allow_permission_bypass,
- created_at, updated_at, last_connected_at, last_disconnected_at
+ created_at, updated_at, last_connected_at, last_disconnected_at,
+ EXISTS (
+ SELECT 1 FROM harness_runtime_connection c
+ WHERE c.harness_id = harnesses.id AND c.lease_expires_at > now()
+ ) AS "has_live_connection!",
+ EXISTS (
+ SELECT 1 FROM harness_runtime_connection c
+ WHERE c.harness_id = harnesses.id
+ ) AS "has_connection_record!"
"#,
harness_id.as_uuid(),
changes.name,
diff --git a/crates/harnesses/src/outbound/pg_harness_repo/tests.rs b/crates/harnesses/src/outbound/pg_harness_repo/tests.rs
index 14238e0805..8f67f61f0f 100644
--- a/crates/harnesses/src/outbound/pg_harness_repo/tests.rs
+++ b/crates/harnesses/src/outbound/pg_harness_repo/tests.rs
@@ -281,7 +281,7 @@ async fn deletion_soft_deletes_and_revokes_tokens(pool: PgPool) {
}
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
-async fn connected_state_follows_the_presence_timestamps(pool: PgPool) {
+async fn connected_state_falls_back_to_timestamps_without_connection_rows(pool: PgPool) {
let repo = PgHarnessRepo::new(pool.clone());
let harness_id = Uuid::new_v4();
sqlx::query!(
@@ -308,6 +308,62 @@ async fn connected_state_follows_the_presence_timestamps(pool: PgPool) {
assert!(!harness.connected);
}
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn connected_state_follows_connection_leases_over_timestamps(pool: PgPool) {
+ let repo = PgHarnessRepo::new(pool.clone());
+ let harness_id = Uuid::new_v4();
+ sqlx::query!(
+ r#"
+ INSERT INTO harnesses (id, name, owner_user_id, created_by, last_connected_at, last_disconnected_at)
+ VALUES ($1, 'mine', $2, $2, now(), now() + interval '1 hour')
+ "#,
+ harness_id,
+ OWNER_ID,
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ let connection_id = Uuid::new_v4();
+ sqlx::query!(
+ r#"
+ INSERT INTO harness_runtime_connection (harness_id, id, replica_id, lease_expires_at)
+ VALUES ($1, $2, $3, now() + interval '30 seconds')
+ "#,
+ harness_id,
+ connection_id,
+ Uuid::new_v4(),
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ let harness_id = HarnessId::new_from_uuid(harness_id);
+
+ // A live lease wins over a later disconnect stamp.
+ let harness = repo.get_harness(harness_id).await.unwrap().unwrap();
+ assert!(harness.connected);
+
+ sqlx::query!(
+ r#"
+ UPDATE harnesses SET last_disconnected_at = NULL;
+ "#,
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ sqlx::query!(
+ r#"UPDATE harness_runtime_connection SET lease_expires_at = now() - interval '1 second'"#,
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+
+ // A crashed replica writes no disconnect; the lapsed lease is what ends it.
+ let harness = repo.get_harness(harness_id).await.unwrap().unwrap();
+ assert!(!harness.connected);
+ let listed = repo.list_visible_harnesses(caller()).await.unwrap();
+ assert!(!listed.iter().any(|harness| harness.connected));
+}
+
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
async fn bound_agents_lists_only_live_agents_of_this_harness(pool: PgPool) {
let repo = PgHarnessRepo::new(pool.clone());
diff --git a/crates/macro_db_client/migrations/20261105153310_create_harness_runtime_connections.sql b/crates/macro_db_client/migrations/20261105153310_create_harness_runtime_connections.sql
new file mode 100644
index 0000000000..095dc264df
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261105153310_create_harness_runtime_connections.sql
@@ -0,0 +1,13 @@
+-- One row per live harness runtime connection, written by the replica holding
+-- its socket. A harness is connected while a row with an unexpired lease exists,
+-- so one connection's close cannot affect another's, whatever order replicas
+-- notice them in, and a replica that crashes takes its connections offline when
+-- its leases lapse.
+CREATE TABLE public.harness_runtime_connection (
+ harness_id uuid NOT NULL REFERENCES public.harnesses(id) ON DELETE CASCADE,
+ id uuid NOT NULL,
+ replica_id uuid NOT NULL,
+ connected_at timestamptz NOT NULL DEFAULT now(),
+ lease_expires_at timestamptz NOT NULL,
+ PRIMARY KEY (harness_id, id)
+);
diff --git a/infra/stacks/monitoring/AGENTS.md b/infra/stacks/monitoring/AGENTS.md
index c7b8d3a759..7f8418c331 100644
--- a/infra/stacks/monitoring/AGENTS.md
+++ b/infra/stacks/monitoring/AGENTS.md
@@ -61,6 +61,8 @@ From `infra/`:
- `monitors/whatsapp-session.ts` reads log events by `telemetry_event`; its
header is the field contract with `services/whatsapp_session_service`.
Renaming a field there silently turns its monitor into missing data.
+- `monitors/harness-presence.ts` reads `harness_presence_disagreement` logged by
+ `crates/harnesses/src/inbound/axum_router.rs`; its header is the field contract.
- `monitors/outlook-sync.ts` reads the health log, sync spans and Graph and
token span fields email-service emits; its header is the field contract.
It finds the Graph and token spans by `@outlook.throttle_origin` and
diff --git a/infra/stacks/monitoring/README.md b/infra/stacks/monitoring/README.md
index aa9c24e694..b0fe6ea039 100644
--- a/infra/stacks/monitoring/README.md
+++ b/infra/stacks/monitoring/README.md
@@ -115,8 +115,9 @@ The gate is that preview:
of the channel), and the GraphQL soup volume, websocket upgrade, gateway
connections and task count monitors their `query` and thresholds (volume
and websocket upgrades also gain recovery thresholds).
-- **90 resources to create**: 70 monitors (31 prod threshold and anomaly, 19 GraphQL soup, 9
- notification retention, 10 WhatsApp session, 1 native call push), 2 webhooks,
+- **92 resources to create**: 72 monitors (31 prod threshold and anomaly, 19 GraphQL soup, 9
+ notification retention, 10 WhatsApp session, 1 native call push, 2 harness
+ presence), 2 webhooks,
7 log-based metrics, 3 span-based metrics and 1 APM retention filter (6, 3
and 1 of them for Outlook sync), the GoAlert synthetic test, the five prod
alert dashboards, and the notification retention dashboard. If the paging
diff --git a/infra/stacks/monitoring/index.ts b/infra/stacks/monitoring/index.ts
index bad01779e0..e514ba60f7 100644
--- a/infra/stacks/monitoring/index.ts
+++ b/infra/stacks/monitoring/index.ts
@@ -21,6 +21,7 @@ import './goalert';
import './monitors/ai-editing';
import './monitors/calls';
import './monitors/graphql-soup';
+import './monitors/harness-presence';
import './monitors/macrodb';
import './monitors/outlook-sync';
import './monitors/queues';
diff --git a/infra/stacks/monitoring/monitors/harness-presence.ts b/infra/stacks/monitoring/monitors/harness-presence.ts
new file mode 100644
index 0000000000..bf66ec23c8
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/harness-presence.ts
@@ -0,0 +1,41 @@
+// Harness presence: whether the service's "connected" agrees with macrod.
+//
+// New monitor, not adopted. It reads a structured log event, keyed by
+// `telemetry_event`, so the fields below are its contract:
+//
+// - `harness_presence_disagreement`, from document-storage-service, at most once
+// a minute per harness: macrod says it holds a runtime connection and the
+// service reads that harness as offline. Fields: `harness`.
+//
+// Ids are opaque harness uuids. Macrod sends the signal only once its connection
+// has been up for a full minute, so a healthy reconnect never reaches it.
+import * as datadog from '@pulumi/datadog';
+import { goalertWebhooks } from '../goalert';
+import { notify, RENOTIFY_MINUTES } from '../notify';
+
+for (const environment of ['dev', 'prod'] as const) {
+ const env = `env:${environment}`;
+
+ new datadog.Monitor(
+ `harness-presence-disagreement-${environment}`,
+ {
+ name: `[${environment.toUpperCase()}] Harness presence: {{harness.name}} online but reported offline`,
+ type: 'log alert',
+ query: `logs("@telemetry_event:harness_presence_disagreement ${env}").index("*").rollup("count").by("@harness").last("15m") > 5`,
+ message: `Macrod for harness {{harness.name}} has held a runtime connection for at least five minutes while the service reports it offline, so the app marks its agents unavailable.
+
+Search @harness:{{harness.name}} @telemetry_event:(harness_runtime_connected OR harness_runtime_disconnected OR harness_runtime_lease_restored). A live connection with no row, or a lease that stopped renewing, is on the replica named in those events. See crates/agent_harness/AGENTS.md.
+
+${notify({ pages: false, environment })}`,
+ monitorThresholds: { critical: '5' },
+ tags: [env, 'service:agent-harness-service'],
+ priority: environment === 'prod' ? '2' : '3',
+ includeTags: true,
+ requireFullWindow: false,
+ renotifyInterval: RENOTIFY_MINUTES,
+ // Daemons older than this signal send nothing; that is not a disagreement.
+ onMissingData: 'resolve',
+ },
+ { dependsOn: goalertWebhooks }
+ );
+}
diff --git a/services/agent_harness_service/src/harness_bindings.rs b/services/agent_harness_service/src/harness_bindings.rs
index dbda857a7a..33051b5f55 100644
--- a/services/agent_harness_service/src/harness_bindings.rs
+++ b/services/agent_harness_service/src/harness_bindings.rs
@@ -3,22 +3,34 @@
//! Composition-root adapters: [`PgHarnessBindings`] answers "which registered
//! harness serves this bot right now" for session binding, and
//! [`PgHarnessPresence`] writes the attach/detach bookkeeping the harness
-//! settings page reads connection state from.
+//! settings page reads connection state from: one leased row per connection, so
+//! replicas that see a harness's sockets end out of order cannot overwrite each
+//! other's answer.
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
+use std::time::Duration;
use agent_client_protocol::schema::v1::SessionId;
-use agent_harness::domain::ports::{HarnessBindings, HarnessPresence};
-use agent_session::domain::model::AgentSessionId;
+use agent_harness::domain::ports::{HarnessBindings, HarnessPresence, RuntimeConnectionId};
+use agent_session::domain::model::{AgentSessionId, ReplicaId};
use bot_id::BotId;
use harness_id::HarnessId;
use sqlx::PgPool;
+use tracing::Instrument as _;
#[cfg(test)]
mod test;
+/// How long a connection counts as live without a renewal. Three missed
+/// renewals: a slow write does not flip a harness offline, and a crashed
+/// replica's connections end within half a minute.
+const RUNTIME_CONNECTION_LEASE: Duration = Duration::from_secs(30);
+
+/// How often a replica renews the leases of the connections it holds.
+pub const RUNTIME_CONNECTION_RENEW_EVERY: Duration = Duration::from_secs(10);
+
/// [`HarnessBindings`] over the `agent_configs` table.
#[derive(Clone)]
pub struct PgHarnessBindings {
@@ -78,46 +90,209 @@ impl HarnessBindings for PgHarnessBindings {
}
}
-/// [`HarnessPresence`] over the `harnesses` table.
+/// Take a harness's presence lock until the transaction ends.
+///
+/// Closing decides whether it was the last live connection by reading its peers'
+/// rows. Without this, two simultaneous closes each count the other as live and
+/// neither stamps the harness. `NO KEY` leaves renewals' foreign key checks free.
+async fn lock_presence(
+ transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+ harness: HarnessId,
+) -> sqlx::Result<()> {
+ sqlx::query_scalar!(
+ r#"SELECT id FROM harnesses WHERE id = $1 FOR NO KEY UPDATE"#,
+ harness.as_uuid(),
+ )
+ .fetch_optional(&mut **transaction)
+ .await?;
+ Ok(())
+}
+
+/// [`HarnessPresence`] over the `harness_runtime_connection` table, kept in step
+/// with the `harnesses` presence timestamps.
pub struct PgHarnessPresence {
pool: PgPool,
+ replica: ReplicaId,
}
impl PgHarnessPresence {
- /// Wrap a pool.
- pub fn new(pool: PgPool) -> Self {
- Self { pool }
+ /// Wrap a pool, writing as `replica`.
+ pub fn new(pool: PgPool, replica: ReplicaId) -> Self {
+ Self { pool, replica }
}
}
impl HarnessPresence for PgHarnessPresence {
- fn connected(self: Arc<Self>, harness: HarnessId) -> Pin<Box<dyn Future<Output = ()> + Send>> {
- Box::pin(async move {
- let result = sqlx::query!(
- r#"UPDATE harnesses SET last_connected_at = now() WHERE id = $1"#,
- harness.as_uuid(),
- )
- .execute(&self.pool)
- .await;
- if let Err(error) = result {
- tracing::error!(error = ?error, %harness, "failed to record harness attach");
+ fn connected(
+ self: Arc<Self>,
+ harness: HarnessId,
+ connection: RuntimeConnectionId,
+ ) -> Pin<Box<dyn Future<Output = ()> + Send>> {
+ let span = tracing::info_span!(
+ "harness.presence.connected",
+ %harness,
+ %connection,
+ replica = %self.replica,
+ );
+ Box::pin(
+ async move {
+ // Lapsed rows go here rather than on a timer: until a harness
+ // dials again they are what says its last replica died.
+ let result = async {
+ let mut transaction = self.pool.begin().await?;
+ lock_presence(&mut transaction, harness).await?;
+ sqlx::query!(
+ r#"
+ WITH reaped AS (
+ DELETE FROM harness_runtime_connection
+ WHERE harness_id = $1 AND lease_expires_at <= now()
+ ), added AS (
+ INSERT INTO harness_runtime_connection
+ (harness_id, id, replica_id, lease_expires_at)
+ VALUES ($1, $2, $3, now() + make_interval(secs => $4))
+ ON CONFLICT (harness_id, id) DO NOTHING
+ )
+ UPDATE harnesses SET last_connected_at = now() WHERE id = $1
+ "#,
+ harness.as_uuid(),
+ connection.as_uuid(),
+ self.replica.as_uuid(),
+ RUNTIME_CONNECTION_LEASE.as_secs_f64(),
+ )
+ .execute(&mut *transaction)
+ .await?;
+ transaction.commit().await
+ }
+ .await;
+ match result {
+ Ok(()) => tracing::info!(
+ telemetry_event = "harness_runtime_connected",
+ %harness,
+ %connection,
+ replica = %self.replica,
+ "recorded a harness runtime connection"
+ ),
+ Err(error) => {
+ tracing::error!(error = ?error, %harness, %connection, "failed to record harness attach");
+ }
+ }
}
- })
+ .instrument(span),
+ )
}
fn disconnected(
self: Arc<Self>,
harness: HarnessId,
+ connection: RuntimeConnectionId,
+ ) -> Pin<Box<dyn Future<Output = ()> + Send>> {
+ let span = tracing::info_span!(
+ "harness.presence.disconnected",
+ %harness,
+ %connection,
+ replica = %self.replica,
+ );
+ Box::pin(
+ async move {
+ // The stamp is what a reader without these rows goes by, so it
+ // moves only when this was the harness's last live connection.
+ let result = async {
+ let mut transaction = self.pool.begin().await?;
+ lock_presence(&mut transaction, harness).await?;
+ let closed = sqlx::query!(
+ r#"
+ WITH removed AS (
+ DELETE FROM harness_runtime_connection
+ WHERE harness_id = $1 AND (id = $2 OR lease_expires_at <= now())
+ RETURNING id, connected_at
+ ), others AS (
+ SELECT count(*) AS live
+ FROM harness_runtime_connection
+ WHERE harness_id = $1 AND id <> $2 AND lease_expires_at > now()
+ ), stamped AS (
+ UPDATE harnesses SET last_disconnected_at = now()
+ WHERE id = $1 AND (SELECT live FROM others) = 0
+ )
+ SELECT
+ (SELECT live FROM others) AS "other_live_connections!",
+ (
+ SELECT (extract(epoch FROM now() - connected_at) * 1000)::bigint
+ FROM removed
+ WHERE id = $2
+ ) AS connected_for_ms
+ "#,
+ harness.as_uuid(),
+ connection.as_uuid(),
+ )
+ .fetch_one(&mut *transaction)
+ .await?;
+ transaction.commit().await?;
+ Ok::<_, sqlx::Error>(closed)
+ }
+ .await;
+ match result {
+ Ok(closed) => tracing::info!(
+ telemetry_event = "harness_runtime_disconnected",
+ %harness,
+ %connection,
+ replica = %self.replica,
+ other_live_connections = closed.other_live_connections,
+ connected_for_ms = closed.connected_for_ms,
+ "recorded a closed harness runtime connection"
+ ),
+ Err(error) => {
+ tracing::error!(error = ?error, %harness, %connection, "failed to record harness detach");
+ }
+ }
+ }
+ .instrument(span),
+ )
+ }
+
+ fn renew(
+ self: Arc<Self>,
+ live: Vec<(HarnessId, RuntimeConnectionId)>,
) -> Pin<Box<dyn Future<Output = ()> + Send>> {
Box::pin(async move {
- let result = sqlx::query!(
- r#"UPDATE harnesses SET last_disconnected_at = now() WHERE id = $1"#,
- harness.as_uuid(),
+ let (harnesses, connections): (Vec<_>, Vec<_>) = live
+ .into_iter()
+ .map(|(harness, connection)| (harness.as_uuid(), connection.as_uuid()))
+ .unzip();
+ // An upsert, so a connection whose attach write failed, or whose
+ // lapsed row was reaped while this process stalled, comes back.
+ let renewed = sqlx::query!(
+ r#"
+ INSERT INTO harness_runtime_connection
+ (harness_id, id, replica_id, lease_expires_at)
+ SELECT live.harness_id, live.id, $3, now() + make_interval(secs => $4)
+ FROM UNNEST($1::uuid[], $2::uuid[]) AS live(harness_id, id)
+ JOIN harnesses ON harnesses.id = live.harness_id
+ ON CONFLICT (harness_id, id) DO UPDATE
+ SET lease_expires_at = EXCLUDED.lease_expires_at
+ RETURNING harness_id, id, (xmax = 0) AS "restored!"
+ "#,
+ &harnesses,
+ &connections,
+ self.replica.as_uuid(),
+ RUNTIME_CONNECTION_LEASE.as_secs_f64(),
)
- .execute(&self.pool)
+ .fetch_all(&self.pool)
.await;
- if let Err(error) = result {
- tracing::error!(error = ?error, %harness, "failed to record harness detach");
+ match renewed {
+ Ok(renewed) => {
+ for row in renewed.into_iter().filter(|row| row.restored) {
+ tracing::warn!(
+ telemetry_event = "harness_runtime_lease_restored",
+ harness = %row.harness_id,
+ connection = %row.id,
+ replica = %self.replica,
+ "a held harness runtime connection had no presence row"
+ );
+ }
+ }
+ Err(error) => {
+ tracing::warn!(error = ?error, replica = %self.replica, "failed to renew harness runtime leases");
+ }
}
})
}
diff --git a/services/agent_harness_service/src/harness_bindings/test.rs b/services/agent_harness_service/src/harness_bindings/test.rs
index 021355c765..afec118e0f 100644
--- a/services/agent_harness_service/src/harness_bindings/test.rs
+++ b/services/agent_harness_service/src/harness_bindings/test.rs
@@ -1,4 +1,6 @@
use super::*;
+use harnesses::domain::ports::HarnessRepo as _;
+use harnesses::outbound::pg_harness_repo::PgHarnessRepo;
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use sqlx::types::Uuid;
@@ -113,3 +115,136 @@ async fn a_deleted_harness_reattaches_nothing(pool: PgPool) {
assert!(found.is_empty());
}
+
+fn presence_on_a_replica(pool: &PgPool) -> Arc<PgHarnessPresence> {
+ Arc::new(PgHarnessPresence::new(pool.clone(), ReplicaId::mint()))
+}
+
+async fn reads_connected(pool: &PgPool, harness: HarnessId) -> bool {
+ PgHarnessRepo::new(pool.clone())
+ .get_harness(harness)
+ .await
+ .unwrap()
+ .unwrap()
+ .connected
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_stale_close_on_another_replica_leaves_a_redialed_harness_connected(pool: PgPool) {
+ let harness = insert_harness(&pool).await;
+ let replica_a = presence_on_a_replica(&pool);
+ let replica_b = presence_on_a_replica(&pool);
+ let old_socket = RuntimeConnectionId::generate();
+ let new_socket = RuntimeConnectionId::generate();
+
+ Arc::clone(&replica_a).connected(harness, old_socket).await;
+ // The old socket goes silent. The daemon redials, reaching the other replica
+ // before the first one notices.
+ Arc::clone(&replica_b).connected(harness, new_socket).await;
+ assert!(reads_connected(&pool, harness).await);
+
+ replica_a.disconnected(harness, old_socket).await;
+
+ assert!(reads_connected(&pool, harness).await);
+
+ // The stamp stays put too, for readers that still go by the timestamps.
+ let stamped = sqlx::query_scalar!(
+ r#"SELECT last_disconnected_at FROM harnesses WHERE id = $1"#,
+ harness.as_uuid(),
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap();
+ assert_eq!(stamped, None);
+
+ replica_b.disconnected(harness, new_socket).await;
+
+ assert!(!reads_connected(&pool, harness).await);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_replica_crash_ends_its_connections_when_their_lease_lapses(pool: PgPool) {
+ let harness = insert_harness(&pool).await;
+ let crashed = presence_on_a_replica(&pool);
+ let connection = RuntimeConnectionId::generate();
+ crashed.connected(harness, connection).await;
+ assert!(reads_connected(&pool, harness).await);
+
+ // A crash writes no disconnect; all that happens is renewals stop.
+ sqlx::query!(
+ r#"UPDATE harness_runtime_connection SET lease_expires_at = now() - interval '1 second'"#,
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ assert!(!reads_connected(&pool, harness).await);
+
+ // The daemon's next dial brings it back and clears the dead row.
+ let successor = presence_on_a_replica(&pool);
+ successor
+ .connected(harness, RuntimeConnectionId::generate())
+ .await;
+ assert!(reads_connected(&pool, harness).await);
+ let rows = sqlx::query_scalar!(
+ r#"SELECT count(*) AS "count!" FROM harness_runtime_connection WHERE harness_id = $1"#,
+ harness.as_uuid(),
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap();
+ assert_eq!(rows, 1);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn renewal_extends_a_lease_and_restores_a_missing_row(pool: PgPool) {
+ let harness = insert_harness(&pool).await;
+ let replica = presence_on_a_replica(&pool);
+ let connection = RuntimeConnectionId::generate();
+ Arc::clone(&replica).connected(harness, connection).await;
+
+ sqlx::query!(
+ r#"UPDATE harness_runtime_connection SET lease_expires_at = now() - interval '1 second'"#,
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ Arc::clone(&replica)
+ .renew(vec![(harness, connection)])
+ .await;
+ assert!(reads_connected(&pool, harness).await);
+
+ sqlx::query!(r#"DELETE FROM harness_runtime_connection"#)
+ .execute(&pool)
+ .await
+ .unwrap();
+ sqlx::query!(r#"UPDATE harnesses SET last_disconnected_at = now() + interval '1 hour'"#)
+ .execute(&pool)
+ .await
+ .unwrap();
+ assert!(!reads_connected(&pool, harness).await);
+
+ replica.renew(vec![(harness, connection)]).await;
+
+ assert!(reads_connected(&pool, harness).await);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn simultaneous_final_closes_leave_a_harness_offline(pool: PgPool) {
+ // Racing writes interleave differently each time, so give them many chances.
+ for _ in 0..25 {
+ let harness = insert_harness(&pool).await;
+ let replica_a = presence_on_a_replica(&pool);
+ let replica_b = presence_on_a_replica(&pool);
+ let first = RuntimeConnectionId::generate();
+ let second = RuntimeConnectionId::generate();
+ Arc::clone(&replica_a).connected(harness, first).await;
+ Arc::clone(&replica_b).connected(harness, second).await;
+
+ tokio::join!(
+ replica_a.disconnected(harness, first),
+ replica_b.disconnected(harness, second),
+ );
+
+ assert!(!reads_connected(&pool, harness).await);
+ }
+}
diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rs
index 36c2aff2ef..81aee2acba 100644
--- a/services/agent_harness_service/src/main.rs
+++ b/services/agent_harness_service/src/main.rs
@@ -136,7 +136,7 @@ use github::outbound::pull_request_diff::GithubPullRequestDiffClient;
use github_pull_requests::domain::service::GithubPullRequestChangesetStore;
use github_pull_requests::outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo;
use github_pull_requests::outbound::s3_patch_store::S3GithubPullRequestPatchStore;
-use harness_bindings::{PgHarnessBindings, PgHarnessPresence};
+use harness_bindings::{PgHarnessBindings, PgHarnessPresence, RUNTIME_CONNECTION_RENEW_EVERY};
use harnesses::outbound::pg_harness_repo::PgHarnessRepo;
use kafka_util::{GroupName, KafkaEventConsumer, consumer_span, record_span_error};
use lexical_client::LexicalClient;
@@ -1283,9 +1283,10 @@ async fn run() -> anyhow::Result<()> {
// the harness loads once it exists below.
let (live_sessions, mut live_session_reports) = tokio::sync::mpsc::unbounded_channel();
let runtimes = RuntimeRegistry::with_presence(
- Arc::new(PgHarnessPresence::new(pool.clone())),
+ Arc::new(PgHarnessPresence::new(pool.clone(), replica)),
live_sessions,
);
+ let presence_runtimes = Arc::clone(&runtimes);
// Read once, here, rather than at every session this deployment opens: a
// URL that names no repository is a misconfiguration of the deployment,
// and refusing it at startup is the difference between one loud failure
@@ -1881,6 +1882,18 @@ async fn run() -> anyhow::Result<()> {
}
});
+ // Keeps vouching for the runtime connections this process holds, including
+ // while it drains: they stay live until their sockets close. When this stops
+ // in a crash, the leases lapse and those harnesses read offline.
+ let presence_renewal = tokio::spawn(async move {
+ let mut ticker = tokio::time::interval(RUNTIME_CONNECTION_RENEW_EVERY);
+ ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+ loop {
+ ticker.tick().await;
+ presence_runtimes.renew_presence().await;
+ }
+ });
+
// Viewers drop a typing indicator nobody refreshes after a few seconds;
// every turn running here keeps its agent's indicator alive until it ends.
let typing_harness = harness.clone();
@@ -2260,6 +2273,7 @@ async fn run() -> anyhow::Result<()> {
trigger.abort();
egress_http.abort();
heartbeat.abort();
+ presence_renewal.abort();
dm_recovery.abort();
typing_heartbeat.abort();
recovery.abort();