Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

fix(pa-daemon): idle passivation never discards state a revival cannot restore (#3206)

Sebastian Müller1ff9d86e14b5
diff --git a/crates/pa-daemon/src/agent_engine/session_engine_impl.rs b/crates/pa-daemon/src/agent_engine/session_engine_impl.rs
index d0356bd214..52e3e8609d 100644
--- a/crates/pa-daemon/src/agent_engine/session_engine_impl.rs
+++ b/crates/pa-daemon/src/agent_engine/session_engine_impl.rs
@@ -26,45 +26,23 @@ impl SessionEngine for AgentSessionEngine {
}
}
- /// The engine-side gate mirror of the whole-worker idle
- /// passivation (TS `canPassivateSession`, session-action-store
- /// :411-419): `true` only when no non-passive descendants hold work
- /// and no active-or-paused scheduled job is registered (the shared
- /// store covers crons AND armed heartbeats — the wake-blind
- /// substitution). This method only answers the gate; the caller
- /// owns the residency decision (the kernel release and the
- /// whole-worker stop each consume it separately).
- fn can_passivate_settled_session(
+ /// The whole-worker idle passivation gate (the
+ /// `idleEvictionMinutes` consumer): the settled gates plus an
+ /// empty RLM child registry — the registry rule's rationale
+ /// lives on the trait method. The kernel release below does
+ /// NOT carry the registry rule: releasing a kernel keeps the
+ /// worker (and its registry) resident.
+ fn can_passivate_worker(
&self,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + '_>> {
Box::pin(async move {
- // `hasNonPassiveDescendants`: a busy descendant keeps the
- // child resident (the TS policy).
- if self.has_unsettled_rlm_work().await {
+ if !self.settled_passivation_gates_pass().await {
return false;
}
- // `hasRegisteredCronJob`: an active or paused scheduled job
- // keeps the worker resident — for the whole-worker idle
- // passivation this gate covers plain cron jobs AND armed
- // heartbeats alike (the shared scheduled-jobs store holds
- // both): the port has no relaunch-on-fire for a stopped
- // worker's jobs, so unlike TS's tier-2 (which evicts
- // cron-armed workers and lets the fire relaunch) the port
- // BLOCKS while any job is armed — the wake-blind
- // substitution, a disclosed deliberate divergence until a
- // relaunch-on-fire port exists. An unwired probe stays open
- // (a store-less embedding never passes the gate).
- let probe = self
- .registered_jobs_probe
- .lock()
- .expect("registered jobs probe lock")
- .clone();
- if let Some(probe) = probe {
- if probe() {
- return false;
- }
+ match self.children.clone() {
+ Some(children) => children.child_identities().await.is_empty(),
+ None => true,
}
- true
})
}
@@ -72,7 +50,7 @@ impl SessionEngine for AgentSessionEngine {
&self,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + '_>> {
Box::pin(async move {
- if !self.can_passivate_settled_session().await {
+ if !self.settled_passivation_gates_pass().await {
return;
}
// The release itself: best-effort (a failed stop leaves
diff --git a/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs b/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs
index 1f54267418..30a2a8aabc 100644
--- a/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs
+++ b/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs
@@ -294,32 +294,37 @@ fn release_settled_child_kernel_defers_to_scheduled_jobs_and_fires_the_release_p
assert_eq!(fired.load(std::sync::atomic::Ordering::SeqCst), 1);
}
+/// The whole-worker idle passivation gate accepts exactly when the
+/// kernel release would fire under the shared settled gates, plus the
+/// worker-only registry rule: a child record keeps the worker resident
+/// (a revival starts with an empty registry) while the kernel release
+/// still fires — releasing a kernel keeps the worker and its registry
+/// resident.
#[test]
-fn can_passivate_settled_session_mirrors_the_release_gates() {
- // The whole-worker idle passivation's engine gate must accept exactly
- // when the kernel release would fire: no unsettled descendants and no
- // registered active-or-paused scheduled job (the shared store covers
- // cron jobs AND armed heartbeats — the wake-blind substitution).
+fn can_passivate_worker_mirrors_the_release_gates_and_adds_the_registry_rule() {
let dir = tempfile::TempDir::new().unwrap();
- let engine = std::sync::Arc::new(bare_engine(dir.path()));
- // No children and no jobs probe: the gates pass.
- assert!(engine
- .runtime
- .block_on(crate::engine::SessionEngine::can_passivate_settled_session(
- &*engine
- )));
- // A jobs probe reporting armed jobs blocks the passivation.
- *engine
- .registered_jobs_probe
- .lock()
- .expect("registered jobs probe lock") = Some(std::sync::Arc::new(|| true));
- assert!(!engine
- .runtime
- .block_on(crate::engine::SessionEngine::can_passivate_settled_session(
- &*engine
- )));
- // The release consumes the same gate: the probe never fires while a
- // job is armed.
+ let engine = std::sync::Arc::new(
+ AgentSessionEngine::new(AgentEngineConfig {
+ cwd: dir.path().to_path_buf(),
+ agent_dir: dir.path().join("agent"),
+ provider: None,
+ model: None,
+ api_key: None,
+ thinking: None,
+ session_dir: None,
+ session_file: None,
+ faux_script: None,
+ supervisor_link: Some(crate::agent_engine::SupervisorLinkConfig {
+ socket_path: dir.path().join("dead.sock"),
+ active_session_id: "parent-session".to_string(),
+ worker_token: "token".to_string(),
+ }),
+ telemetry_disabled: None,
+ cron_store: None,
+ queued_steering_probe: None,
+ })
+ .unwrap(),
+ );
let fired = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let probe_fired = std::sync::Arc::clone(&fired);
*engine
@@ -329,10 +334,71 @@ fn can_passivate_settled_session_mirrors_the_release_gates() {
probe_fired.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Box::pin(std::future::ready(()))
}));
+ // An empty registry with nothing armed: the gate passes.
+ assert!(engine
+ .runtime
+ .block_on(crate::engine::SessionEngine::can_passivate_worker(&*engine)));
+ // A live background bash handle blocks the passivation (the kernel
+ // snapshot cannot resurrect a live process), and the release consumes
+ // the same gate: the probe never fires while a handle runs.
+ *engine
+ .background_bash_probe
+ .lock()
+ .expect("background bash probe lock") = Some(std::sync::Arc::new(|| true));
+ assert!(!engine
+ .runtime
+ .block_on(crate::engine::SessionEngine::can_passivate_worker(&*engine)));
engine
.runtime
.block_on(crate::engine::SessionEngine::release_settled_child_kernel(
&*engine,
));
assert_eq!(fired.load(std::sync::atomic::Ordering::SeqCst), 0);
+ *engine
+ .background_bash_probe
+ .lock()
+ .expect("background bash probe lock") = None;
+ // A jobs probe reporting armed jobs blocks the passivation the same
+ // way.
+ *engine
+ .registered_jobs_probe
+ .lock()
+ .expect("registered jobs probe lock") = Some(std::sync::Arc::new(|| true));
+ assert!(!engine
+ .runtime
+ .block_on(crate::engine::SessionEngine::can_passivate_worker(&*engine)));
+ engine
+ .runtime
+ .block_on(crate::engine::SessionEngine::release_settled_child_kernel(
+ &*engine,
+ ));
+ assert_eq!(fired.load(std::sync::atomic::Ordering::SeqCst), 0);
+ *engine
+ .registered_jobs_probe
+ .lock()
+ .expect("registered jobs probe lock") = None;
+ // A settled child record keeps the worker resident (a revival starts
+ // with an empty registry) while the kernel release still fires: the
+ // registry rule is whole-worker only.
+ let children = engine.children.clone().expect("children registry");
+ engine.runtime.block_on(async {
+ children
+ .push_test_child(crate::rlm_children::RlmChildIdentity {
+ rlm_child_id: "child-1".to_string(),
+ active_session_id: "child-session".to_string(),
+ session_id: None,
+ session_name: "worker-1".to_string(),
+ })
+ .await;
+ children.settle_test_child("child-session").await;
+ });
+ assert!(!engine
+ .runtime
+ .block_on(crate::engine::SessionEngine::can_passivate_worker(&*engine)));
+ engine
+ .runtime
+ .block_on(crate::engine::SessionEngine::release_settled_child_kernel(
+ &*engine,
+ ));
+ assert_eq!(fired.load(std::sync::atomic::Ordering::SeqCst), 1);
}
diff --git a/crates/pa-daemon/src/engine.rs b/crates/pa-daemon/src/engine.rs
index 9b80970817..0fb3da4d58 100644
--- a/crates/pa-daemon/src/engine.rs
+++ b/crates/pa-daemon/src/engine.rs
@@ -74,23 +74,25 @@ pub trait SessionEngine: Send + Sync {
/// fires this best-effort from its park arm once the worker core
/// proved the parent-owned, unattached, unqueued idle state;
/// engines that cannot release (scripted harness engines, engines
- /// without a kernel or with unsettled descendants or registered
- /// scheduled jobs) no-op and the child stays resident.
+ /// without a kernel, or engines whose settled gates fail) no-op
+ /// and the child stays resident.
fn release_settled_child_kernel(
&self,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + '_>> {
Box::pin(std::future::ready(()))
}
- /// Whether the settled-child passivation gates pass (TS #2483's
- /// `canPassivateSettledSession` minus the release itself): no
- /// unsettled RLM descendants and no registered active-or-paused
- /// scheduled job. The whole-worker idle passivation (the
- /// `idleEvictionMinutes` consumer) re-checks these engine-side gates
- /// before asking the supervisor for the graceful stop. The default
- /// `false` keeps scripted harness engines and kernel-less embeddings
- /// resident — the same conservative arm as the release default.
- fn can_passivate_settled_session(
+ /// Whether the whole-worker idle passivation gates pass (the
+ /// `idleEvictionMinutes` consumer re-checks them before asking the
+ /// supervisor for the graceful stop): the engine's settled gates
+ /// plus an empty RLM child registry. The registry rule holds
+ /// because the port does not rebuild a revived session's subagent
+ /// registry from the spawn ledger (TS `listPassiveRlmSubagents`
+ /// does), so the stop would discard state the revival cannot
+ /// restore. The default `false` keeps scripted harness engines and
+ /// kernel-less embeddings resident — the same conservative arm as
+ /// the release default.
+ fn can_passivate_worker(
&self,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + '_>> {
Box::pin(std::future::ready(false))
diff --git a/crates/pa-daemon/src/goal_continuation.rs b/crates/pa-daemon/src/goal_continuation.rs
index c84b7d274c..e99abbefcd 100644
--- a/crates/pa-daemon/src/goal_continuation.rs
+++ b/crates/pa-daemon/src/goal_continuation.rs
@@ -304,6 +304,32 @@ impl AgentSessionEngine {
.is_some_and(|probe| probe())
}
+ /// The settled passivation gates shared by the settled-child kernel
+ /// release and the whole-worker idle passivation (TS #2483's
+ /// `canPassivateSettledSession`): no unsettled RLM descendant work,
+ /// no live background `bash()` handle (a kernel snapshot cannot
+ /// resurrect a live process — TS `isSessionActive`'s
+ /// `hasBackgroundWork` arm), and no registered active-or-paused
+ /// scheduled job. The jobs gate covers plain cron jobs AND armed
+ /// heartbeats alike (the shared scheduled-jobs store holds both):
+ /// the port has no relaunch-on-fire for a stopped worker's jobs, so
+ /// unlike TS's tier-2 (which evicts cron-armed workers and lets the
+ /// fire relaunch) the port BLOCKS while any job is armed — the
+ /// wake-blind substitution, a disclosed deliberate divergence until
+ /// a relaunch-on-fire port exists. An unwired jobs probe passes (an
+ /// engine without a scheduled-jobs store has no job to protect).
+ pub(crate) async fn settled_passivation_gates_pass(&self) -> bool {
+ if self.has_unsettled_rlm_work().await || self.has_live_background_bash_handles() {
+ return false;
+ }
+ let probe = self
+ .registered_jobs_probe
+ .lock()
+ .expect("registered jobs probe lock")
+ .clone();
+ !probe.is_some_and(|probe| probe())
+ }
+
/// The worker's session-input probe: `true` while queued user work or
/// the queued-input suspension owns the next turn boundary. An
/// unwired probe (engine without a worker) answers `false`.
diff --git a/crates/pa-daemon/src/rlm_children.rs b/crates/pa-daemon/src/rlm_children.rs
index ceca31aca6..73b6e6de66 100644
--- a/crates/pa-daemon/src/rlm_children.rs
+++ b/crates/pa-daemon/src/rlm_children.rs
@@ -607,6 +607,19 @@ impl SupervisorChildSessions {
}
}
+ /// Test seam: settle a pushed child record (the passivation-gate
+ /// tests exercise a registry that holds only settled children).
+ #[cfg(test)]
+ pub(crate) async fn settle_test_child(&self, child_active_session_id: &str) {
+ let children = self.inner.children.lock().await;
+ for record in children.iter() {
+ let mut record = record.lock().await;
+ if record.active_session_id == child_active_session_id {
+ record.settled_status = Some("done");
+ }
+ }
+ }
+
/// Test seam: admit one child record without the supervisor round trip
/// (the controller tests exercise the family join on registry state).
#[cfg(test)]
diff --git a/crates/pa-daemon/src/worker/turn.rs b/crates/pa-daemon/src/worker/turn.rs
index 0778a34d72..476e58eca0 100644
--- a/crates/pa-daemon/src/worker/turn.rs
+++ b/crates/pa-daemon/src/worker/turn.rs
@@ -234,11 +234,11 @@ impl TurnRunner {
/// pending-prompt-admissions arm) and the setting is a live
/// threshold; `None` otherwise (roots, attached children, `"off"`,
/// and any state the engine gates would reject stay parked without
- /// a timer). The engine-side passivation gates
- /// (unsettled descendants, registered active-or-paused scheduled
- /// jobs) are re-checked at the fire inside
- /// [`Self::maybe_request_idle_passivation`] — the fresh-snapshot
- /// fence — so this window only decides whether to arm.
+ /// a timer). The engine gate
+ /// (`SessionEngine::can_passivate_worker`) is re-checked at the
+ /// fire inside [`Self::maybe_request_idle_passivation`] — the
+ /// fresh-snapshot fence — so this window only decides whether to
+ /// arm.
pub(super) fn idle_passivation_window(&self) -> Option<std::time::Duration> {
let (rlm_depth, attached, compacting, shutdown, queued, last_activity, cwd) = {
let core = self.core.lock().unwrap();
@@ -341,10 +341,10 @@ impl TurnRunner {
if crate::util::now_ms().saturating_sub(last_activity) < minutes.saturating_mul(60_000) {
return;
}
- // The engine-side gates (no unsettled descendants, no registered
- // active-or-paused scheduled job): a parked child with either
- // stays resident.
- if !self.engine.can_passivate_settled_session().await {
+ // The engine gate's one definition lives with the engine
+ // (`SessionEngine::can_passivate_worker`): a parked worker
+ // failing it stays resident.
+ if !self.engine.can_passivate_worker().await {
return;
}
// The post-await revalidation (the fresh bots' race findings):