Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix: quiet managed heartbeat outages

AkitaOnRails886128fd3891
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e6f18606..49fec6a6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+- Managed workstream heartbeats now bound each server request and condense an
+ outage into one short notice plus one recovery notice. Active launchers keep
+ the lease-safe 30-second retry cadence without printing the same timeout on
+ every attempt, and may renew their original run after a longer outage unless
+ another launcher has already claimed the workstream. (#311)
+
## [1.20.1] - 2026-07-30
### Fixed
diff --git a/crates/ai-memory-cli/src/commands/run.rs b/crates/ai-memory-cli/src/commands/run.rs
index 574da5ef..3d7881ff 100644
--- a/crates/ai-memory-cli/src/commands/run.rs
+++ b/crates/ai-memory-cli/src/commands/run.rs
@@ -30,6 +30,7 @@ use crate::http_client::{
};
const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30);
+const HEARTBEAT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
const PREPARE_BUSY_RETRY_WINDOW: Duration = Duration::from_secs(5);
const PREPARE_BUSY_RETRY_INTERVAL: Duration = Duration::from_millis(250);
const IMPORT_BATCH_EVENTS: usize = 400;
@@ -50,6 +51,25 @@ struct AutoSessionCandidate {
session: NativeSessionCandidate,
}
+#[derive(Debug, Default)]
+struct HeartbeatHealth {
+ consecutive_failures: u64,
+}
+
+impl HeartbeatHealth {
+ fn record_failure(&mut self) -> bool {
+ let first = self.consecutive_failures == 0;
+ self.consecutive_failures = self.consecutive_failures.saturating_add(1);
+ first
+ }
+
+ fn record_success(&mut self) -> bool {
+ let recovered = self.consecutive_failures > 0;
+ self.consecutive_failures = 0;
+ recovered
+ }
+}
+
/// Run one native harness and return its exact process exit code.
pub async fn run(config: &Config, args: RunArgs) -> Result<i32> {
let cwd = std::env::current_dir().context("getting managed run working directory")?;
@@ -369,13 +389,16 @@ pub async fn run(config: &Config, args: RunArgs) -> Result<i32> {
let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL);
heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
heartbeat.tick().await;
+ let mut heartbeat_health = HeartbeatHealth::default();
let status = loop {
tokio::select! {
result = child.wait() => break acquired_try!(result.context("waiting for managed harness")),
_ = heartbeat.tick() => {
- if let Err(error) = post_empty(&endpoint, &format!("{run_path}/heartbeat")).await {
- eprintln!("ai-memory: managed workstream heartbeat failed: {error}");
- }
+ let _ = send_managed_heartbeat(
+ &endpoint,
+ &run_path,
+ &mut heartbeat_health,
+ ).await;
}
}
};
@@ -454,6 +477,43 @@ async fn capture_interrupts(interrupted: Arc<AtomicBool>) {
}
}
+async fn send_managed_heartbeat(
+ endpoint: &ServerEndpoint,
+ run_path: &str,
+ health: &mut HeartbeatHealth,
+) -> Result<()> {
+ send_managed_heartbeat_with_timeout(endpoint, run_path, health, HEARTBEAT_REQUEST_TIMEOUT).await
+}
+
+async fn send_managed_heartbeat_with_timeout(
+ endpoint: &ServerEndpoint,
+ run_path: &str,
+ health: &mut HeartbeatHealth,
+ request_timeout: Duration,
+) -> Result<()> {
+ let path = format!("{run_path}/heartbeat");
+ let result = tokio::time::timeout(request_timeout, post_empty(endpoint, &path))
+ .await
+ .map_err(|_| anyhow!("request timed out after {request_timeout:?}"))
+ .and_then(|result| result);
+ match &result {
+ Ok(()) if health.record_success() => {
+ eprintln!(
+ "ai-memory: server connection restored; managed workstream heartbeat resumed"
+ );
+ }
+ Ok(()) => {}
+ Err(error) if health.record_failure() => {
+ tracing::debug!(error = %error, "managed workstream heartbeat became unavailable");
+ eprintln!(
+ "ai-memory: server unavailable; managed workstream heartbeat will retry quietly"
+ );
+ }
+ Err(_) => {}
+ }
+ result
+}
+
async fn cancel_managed_run_after_failure(endpoint: &ServerEndpoint, run_path: &str) {
if let Err(error) = post_empty_with_retry(
endpoint,
@@ -816,19 +876,18 @@ async fn choose_native_session_interactive(
let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL);
heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
heartbeat.tick().await;
+ let mut heartbeat_health = HeartbeatHealth::default();
let selection = loop {
tokio::select! {
result = &mut chooser => {
break result.context("waiting for the native session choice")?;
}
_ = heartbeat.tick() => {
- if let Err(error) = post_empty(endpoint, &format!("{run_path}/heartbeat")).await {
- eprintln!("ai-memory: managed workstream heartbeat failed: {error}");
- }
+ let _ = send_managed_heartbeat(endpoint, run_path, &mut heartbeat_health).await;
}
}
};
- post_empty(endpoint, &format!("{run_path}/heartbeat"))
+ send_managed_heartbeat(endpoint, run_path, &mut heartbeat_health)
.await
.context(
"renewing the managed workstream after session selection; the agent was not started",
@@ -1181,6 +1240,64 @@ mod tests {
assert_eq!(lease_owner_label(None, 42), "localhost:42");
}
+ #[test]
+ fn heartbeat_health_reports_each_outage_and_recovery_once() {
+ let mut health = HeartbeatHealth::default();
+
+ assert!(health.record_failure());
+ assert!(!health.record_failure());
+ assert!(!health.record_failure());
+ assert!(health.record_success());
+ assert!(!health.record_success());
+ assert!(health.record_failure());
+ }
+
+ #[tokio::test]
+ async fn managed_heartbeat_times_out_and_recovers() {
+ let attempts = Arc::new(AtomicUsize::new(0));
+ let handler_attempts = Arc::clone(&attempts);
+ let app = Router::new().route(
+ "/workstream/runs/{run_id}/heartbeat",
+ post(move || {
+ let attempts = Arc::clone(&handler_attempts);
+ async move {
+ if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ }
+ StatusCode::NO_CONTENT
+ }
+ }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ let endpoint = ServerEndpoint::from_pair(Some(format!("http://{address}")), None);
+ let mut health = HeartbeatHealth::default();
+
+ assert!(
+ send_managed_heartbeat_with_timeout(
+ &endpoint,
+ "/workstream/runs/test",
+ &mut health,
+ Duration::from_millis(10),
+ )
+ .await
+ .is_err()
+ );
+ assert_eq!(health.consecutive_failures, 1);
+ send_managed_heartbeat_with_timeout(
+ &endpoint,
+ "/workstream/runs/test",
+ &mut health,
+ Duration::from_secs(1),
+ )
+ .await
+ .unwrap();
+ assert_eq!(health.consecutive_failures, 0);
+
+ server.abort();
+ }
+
#[tokio::test]
async fn prepare_waits_for_a_previous_launcher_to_finish() {
let attempts = Arc::new(AtomicUsize::new(0));
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index 09432615..b1d27ce2 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -4419,7 +4419,7 @@ mod tests {
}
#[tokio::test]
- async fn managed_run_heartbeat_extends_only_a_live_lease() {
+ async fn managed_run_heartbeat_extends_only_an_active_run() {
let tmp = TempDir::new().unwrap();
let store = Store::open(tmp.path()).unwrap();
let (ws, proj) = open_managed_scope(&store, "managed-heartbeat").await;
@@ -4444,10 +4444,11 @@ mod tests {
"heartbeat must extend the lease"
);
- // A lease that already lapsed cannot be revived by a heartbeat.
+ // A still-active launcher can recover after the server was unavailable
+ // for longer than one lease window.
set_managed_run_lease(store.db_path(), run.run_id, 1);
assert!(
- !store
+ store
.writer
.heartbeat_managed_run(run.run_id)
.await
@@ -4477,6 +4478,34 @@ mod tests {
.unwrap()
);
+ // Once another prepare transaction has claimed the workstream, the
+ // superseded run is terminal and cannot displace the replacement.
+ let superseded = store
+ .writer
+ .prepare_workstream_run(managed_prepare_input(ws, proj, "test:superseded"))
+ .await
+ .unwrap();
+ set_managed_run_lease(store.db_path(), superseded.run_id, 1);
+ let replacement = store
+ .writer
+ .prepare_workstream_run(managed_prepare_input(ws, proj, "test:replacement"))
+ .await
+ .unwrap();
+ assert!(
+ !store
+ .writer
+ .heartbeat_managed_run(superseded.run_id)
+ .await
+ .unwrap()
+ );
+ assert!(
+ store
+ .writer
+ .cancel_managed_run(replacement.run_id)
+ .await
+ .unwrap()
+ );
+
// Nor cancelled runs.
let cancelled = store
.writer
diff --git a/crates/ai-memory-store/src/workstream.rs b/crates/ai-memory-store/src/workstream.rs
index a26e2bc3..6f70b481 100644
--- a/crates/ai-memory-store/src/workstream.rs
+++ b/crates/ai-memory-store/src/workstream.rs
@@ -409,13 +409,17 @@ fn validate_workstream_name(name: &str) -> StoreResult<()> {
Ok(())
}
-/// Extend the lease for a live managed run.
+/// Extend the lease for an active managed run.
+///
+/// The timestamp may have lapsed during a server outage. It remains renewable
+/// until another prepare transaction atomically expires this row and grants a
+/// replacement run. Finished, cancelled, and superseded runs stay terminal.
pub(crate) fn heartbeat(conn: &mut Connection, run_id: ManagedRunId) -> StoreResult<bool> {
let now = Timestamp::now().as_microsecond();
let changed = conn.execute(
"UPDATE managed_runs SET lease_expires_at = ?1 \
- WHERE id = ?2 AND state = 'active' AND lease_expires_at > ?3",
- params![now + LEASE_MICROS, run_id.as_bytes(), now],
+ WHERE id = ?2 AND state = 'active'",
+ params![now + LEASE_MICROS, run_id.as_bytes()],
)?;
Ok(changed > 0)
}
diff --git a/docs/managed-workstreams.md b/docs/managed-workstreams.md
index 9daa8a1c..2899b128 100644
--- a/docs/managed-workstreams.md
+++ b/docs/managed-workstreams.md
@@ -281,6 +281,16 @@ the conflict remains and concurrent writers are still rejected. Terminal
interrupts continue to reach the child while the parent stays alive to finish
or cancel the run.
+While the harness or native-session selector is open, a temporary server outage
+produces one short notice instead of printing every failed heartbeat. The
+launcher keeps probing every 30 seconds with a 10-second request timeout so the
+90-second lease stays safe across ordinary server restarts. Repeated failures
+are quiet; when the server responds again, one recovery notice confirms that
+heartbeats resumed. The native harness remains usable throughout the outage.
+If the outage exceeds one lease window, the original launcher may renew its run
+only while no newer launcher has claimed the workstream. A replacement prepare,
+cancel, finish, or destructive operation remains terminal for the old run.
+
If the client is terminated without cleanup, such as with `kill -9`, its lease
expires within 90 seconds. A later managed run starts from the last committed
adapter cursor, so already linked native sessions can import the missing tail