Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix: quiet managed heartbeat outages
AkitaOnRails886128fd3891
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e6f18606..49fec6a6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed+- Managed workstream heartbeats now bound each server request and condense an+ outage into one short notice plus one recovery notice. Active launchers keep+ the lease-safe 30-second retry cadence without printing the same timeout on+ every attempt, and may renew their original run after a longer outage unless+ another launcher has already claimed the workstream. (#311)+## [1.20.1] - 2026-07-30### Fixeddiff --git a/crates/ai-memory-cli/src/commands/run.rs b/crates/ai-memory-cli/src/commands/run.rsindex 574da5ef..3d7881ff 100644--- a/crates/ai-memory-cli/src/commands/run.rs+++ b/crates/ai-memory-cli/src/commands/run.rs@@ -30,6 +30,7 @@ use crate::http_client::{};const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30);+const HEARTBEAT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);const PREPARE_BUSY_RETRY_WINDOW: Duration = Duration::from_secs(5);const PREPARE_BUSY_RETRY_INTERVAL: Duration = Duration::from_millis(250);const IMPORT_BATCH_EVENTS: usize = 400;@@ -50,6 +51,25 @@ struct AutoSessionCandidate {session: NativeSessionCandidate,}+#[derive(Debug, Default)]+struct HeartbeatHealth {+ consecutive_failures: u64,+}++impl HeartbeatHealth {+ fn record_failure(&mut self) -> bool {+ let first = self.consecutive_failures == 0;+ self.consecutive_failures = self.consecutive_failures.saturating_add(1);+ first+ }++ fn record_success(&mut self) -> bool {+ let recovered = self.consecutive_failures > 0;+ self.consecutive_failures = 0;+ recovered+ }+}+/// Run one native harness and return its exact process exit code.pub async fn run(config: &Config, args: RunArgs) -> Result<i32> {let cwd = std::env::current_dir().context("getting managed run working directory")?;@@ -369,13 +389,16 @@ pub async fn run(config: &Config, args: RunArgs) -> Result<i32> {let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL);heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);heartbeat.tick().await;+ let mut heartbeat_health = HeartbeatHealth::default();let status = loop {tokio::select! {result = child.wait() => break acquired_try!(result.context("waiting for managed harness")),_ = heartbeat.tick() => {- if let Err(error) = post_empty(&endpoint, &format!("{run_path}/heartbeat")).await {- eprintln!("ai-memory: managed workstream heartbeat failed: {error}");- }+ let _ = send_managed_heartbeat(+ &endpoint,+ &run_path,+ &mut heartbeat_health,+ ).await;}}};@@ -454,6 +477,43 @@ async fn capture_interrupts(interrupted: Arc<AtomicBool>) {}}+async fn send_managed_heartbeat(+ endpoint: &ServerEndpoint,+ run_path: &str,+ health: &mut HeartbeatHealth,+) -> Result<()> {+ send_managed_heartbeat_with_timeout(endpoint, run_path, health, HEARTBEAT_REQUEST_TIMEOUT).await+}++async fn send_managed_heartbeat_with_timeout(+ endpoint: &ServerEndpoint,+ run_path: &str,+ health: &mut HeartbeatHealth,+ request_timeout: Duration,+) -> Result<()> {+ let path = format!("{run_path}/heartbeat");+ let result = tokio::time::timeout(request_timeout, post_empty(endpoint, &path))+ .await+ .map_err(|_| anyhow!("request timed out after {request_timeout:?}"))+ .and_then(|result| result);+ match &result {+ Ok(()) if health.record_success() => {+ eprintln!(+ "ai-memory: server connection restored; managed workstream heartbeat resumed"+ );+ }+ Ok(()) => {}+ Err(error) if health.record_failure() => {+ tracing::debug!(error = %error, "managed workstream heartbeat became unavailable");+ eprintln!(+ "ai-memory: server unavailable; managed workstream heartbeat will retry quietly"+ );+ }+ Err(_) => {}+ }+ result+}+async fn cancel_managed_run_after_failure(endpoint: &ServerEndpoint, run_path: &str) {if let Err(error) = post_empty_with_retry(endpoint,@@ -816,19 +876,18 @@ async fn choose_native_session_interactive(let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL);heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);heartbeat.tick().await;+ let mut heartbeat_health = HeartbeatHealth::default();let selection = loop {tokio::select! {result = &mut chooser => {break result.context("waiting for the native session choice")?;}_ = heartbeat.tick() => {- if let Err(error) = post_empty(endpoint, &format!("{run_path}/heartbeat")).await {- eprintln!("ai-memory: managed workstream heartbeat failed: {error}");- }+ let _ = send_managed_heartbeat(endpoint, run_path, &mut heartbeat_health).await;}}};- post_empty(endpoint, &format!("{run_path}/heartbeat"))+ send_managed_heartbeat(endpoint, run_path, &mut heartbeat_health).await.context("renewing the managed workstream after session selection; the agent was not started",@@ -1181,6 +1240,64 @@ mod tests {assert_eq!(lease_owner_label(None, 42), "localhost:42");}+ #[test]+ fn heartbeat_health_reports_each_outage_and_recovery_once() {+ let mut health = HeartbeatHealth::default();++ assert!(health.record_failure());+ assert!(!health.record_failure());+ assert!(!health.record_failure());+ assert!(health.record_success());+ assert!(!health.record_success());+ assert!(health.record_failure());+ }++ #[tokio::test]+ async fn managed_heartbeat_times_out_and_recovers() {+ let attempts = Arc::new(AtomicUsize::new(0));+ let handler_attempts = Arc::clone(&attempts);+ let app = Router::new().route(+ "/workstream/runs/{run_id}/heartbeat",+ post(move || {+ let attempts = Arc::clone(&handler_attempts);+ async move {+ if attempts.fetch_add(1, Ordering::SeqCst) == 0 {+ tokio::time::sleep(Duration::from_millis(100)).await;+ }+ StatusCode::NO_CONTENT+ }+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let address = listener.local_addr().unwrap();+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ let endpoint = ServerEndpoint::from_pair(Some(format!("http://{address}")), None);+ let mut health = HeartbeatHealth::default();++ assert!(+ send_managed_heartbeat_with_timeout(+ &endpoint,+ "/workstream/runs/test",+ &mut health,+ Duration::from_millis(10),+ )+ .await+ .is_err()+ );+ assert_eq!(health.consecutive_failures, 1);+ send_managed_heartbeat_with_timeout(+ &endpoint,+ "/workstream/runs/test",+ &mut health,+ Duration::from_secs(1),+ )+ .await+ .unwrap();+ assert_eq!(health.consecutive_failures, 0);++ server.abort();+ }+#[tokio::test]async fn prepare_waits_for_a_previous_launcher_to_finish() {let attempts = Arc::new(AtomicUsize::new(0));diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex 09432615..b1d27ce2 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -4419,7 +4419,7 @@ mod tests {}#[tokio::test]- async fn managed_run_heartbeat_extends_only_a_live_lease() {+ async fn managed_run_heartbeat_extends_only_an_active_run() {let tmp = TempDir::new().unwrap();let store = Store::open(tmp.path()).unwrap();let (ws, proj) = open_managed_scope(&store, "managed-heartbeat").await;@@ -4444,10 +4444,11 @@ mod tests {"heartbeat must extend the lease");- // A lease that already lapsed cannot be revived by a heartbeat.+ // A still-active launcher can recover after the server was unavailable+ // for longer than one lease window.set_managed_run_lease(store.db_path(), run.run_id, 1);assert!(- !store+ store.writer.heartbeat_managed_run(run.run_id).await@@ -4477,6 +4478,34 @@ mod tests {.unwrap());+ // Once another prepare transaction has claimed the workstream, the+ // superseded run is terminal and cannot displace the replacement.+ let superseded = store+ .writer+ .prepare_workstream_run(managed_prepare_input(ws, proj, "test:superseded"))+ .await+ .unwrap();+ set_managed_run_lease(store.db_path(), superseded.run_id, 1);+ let replacement = store+ .writer+ .prepare_workstream_run(managed_prepare_input(ws, proj, "test:replacement"))+ .await+ .unwrap();+ assert!(+ !store+ .writer+ .heartbeat_managed_run(superseded.run_id)+ .await+ .unwrap()+ );+ assert!(+ store+ .writer+ .cancel_managed_run(replacement.run_id)+ .await+ .unwrap()+ );+// Nor cancelled runs.let cancelled = store.writerdiff --git a/crates/ai-memory-store/src/workstream.rs b/crates/ai-memory-store/src/workstream.rsindex a26e2bc3..6f70b481 100644--- a/crates/ai-memory-store/src/workstream.rs+++ b/crates/ai-memory-store/src/workstream.rs@@ -409,13 +409,17 @@ fn validate_workstream_name(name: &str) -> StoreResult<()> {Ok(())}-/// Extend the lease for a live managed run.+/// Extend the lease for an active managed run.+///+/// The timestamp may have lapsed during a server outage. It remains renewable+/// until another prepare transaction atomically expires this row and grants a+/// replacement run. Finished, cancelled, and superseded runs stay terminal.pub(crate) fn heartbeat(conn: &mut Connection, run_id: ManagedRunId) -> StoreResult<bool> {let now = Timestamp::now().as_microsecond();let changed = conn.execute("UPDATE managed_runs SET lease_expires_at = ?1 \- WHERE id = ?2 AND state = 'active' AND lease_expires_at > ?3",- params![now + LEASE_MICROS, run_id.as_bytes(), now],+ WHERE id = ?2 AND state = 'active'",+ params![now + LEASE_MICROS, run_id.as_bytes()],)?;Ok(changed > 0)}diff --git a/docs/managed-workstreams.md b/docs/managed-workstreams.mdindex 9daa8a1c..2899b128 100644--- a/docs/managed-workstreams.md+++ b/docs/managed-workstreams.md@@ -281,6 +281,16 @@ the conflict remains and concurrent writers are still rejected. Terminalinterrupts continue to reach the child while the parent stays alive to finishor cancel the run.+While the harness or native-session selector is open, a temporary server outage+produces one short notice instead of printing every failed heartbeat. The+launcher keeps probing every 30 seconds with a 10-second request timeout so the+90-second lease stays safe across ordinary server restarts. Repeated failures+are quiet; when the server responds again, one recovery notice confirms that+heartbeats resumed. The native harness remains usable throughout the outage.+If the outage exceeds one lease window, the original launcher may renew its run+only while no newer launcher has claimed the workstream. A replacement prepare,+cancel, finish, or destructive operation remains terminal for the old run.+If the client is terminated without cleanup, such as with `kill -9`, its leaseexpires within 90 seconds. A later managed run starts from the last committedadapter cursor, so already linked native sessions can import the missing tail