A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
//! The update flow's graceful-stop driver (spec §5 `Stopping`, §9 budgets,
//! invariant I3): workers stop by an acked request and then exiting,
//! each within its budget; a miss ABANDONS the update — the supervisor
//! returns to `Serving` — and no session is ever killed mid-run.
use std::sync::Arc;
use std::time::Duration;
use anyhow::Result;
use futures::future::join_all;
use pa_types::daemon::UpdateTimeoutBudget;
use crate::registry::ResidentWorker;
/// TS `UPDATE_RESTART_WORKER_REQUEST_TIMEOUT_MS` parity: the per-worker cap
/// for update RPCs, always bounded by the remaining prepare deadline at the call site.
pub(crate) const WORKER_REQUEST_TIMEOUT_MS: u64 = 90_000;
/// One worker's stop verdict.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum WorkerStopVerdict {
/// The worker acked the graceful stop and exited in budget.
Stopped,
/// The worker never acked (or never exited) in budget: the update is
/// abandoned, the session keeps running untouched.
Refused,
}
/// The supervisor's transport to its workers, abstracted so the stop
/// budget logic is unit-testable against a fake worker that never acks.
///
/// Contract (spec §9 `Stopping`): `request_shutdown` is the acked request bounded by
/// `worker_stop_ms`; a success means a flush before replying and gets `worker_stop_extension_ms`
/// of exit wait; `wait_exit` true when gone by the deadline.
pub(crate) trait WorkerStopTransport {
/// Send the acked graceful-stop request; `Err` on timeout, disconnect, or refusal.
fn request_shutdown(
&self,
resident: &Arc<ResidentWorker>,
timeout: Duration,
) -> impl std::future::Future<Output = Result<()>> + Send;
/// Wait until the worker's process is gone or the timeout passes (`true` = exited).
fn wait_exit(
&self,
resident: &Arc<ResidentWorker>,
timeout: Duration,
) -> impl std::future::Future<Output = bool> + Send;
}
/// Stop every resident worker gracefully, concurrently, each within its own budget (spec §9: 30 s
/// request budget, +30 s exit window); a miss abandons the update for that verdict (never a kill).
pub(crate) async fn stop_workers_gracefully<T: WorkerStopTransport + Sync>(
transport: &T,
residents: &[Arc<ResidentWorker>],
budget: &UpdateTimeoutBudget,
) -> Vec<(String, WorkerStopVerdict)> {
join_all(residents.iter().map(|resident| {
let resident = Arc::clone(resident);
async move {
let verdict = stop_one(transport, &resident, budget).await;
(resident.worker_id.clone(), verdict)
}
}))
.await
}
async fn stop_one<T: WorkerStopTransport + Sync>(
transport: &T,
resident: &Arc<ResidentWorker>,
budget: &UpdateTimeoutBudget,
) -> WorkerStopVerdict {
let request_budget = Duration::from_millis(budget.worker_stop_ms);
if transport
.request_shutdown(resident, request_budget)
.await
.is_err()
{
return WorkerStopVerdict::Refused;
}
// The ack means the flush already happened; the exit window is the
// spec's extension budget.
let exit_budget = Duration::from_millis(budget.worker_stop_extension_ms);
if transport.wait_exit(resident, exit_budget).await {
WorkerStopVerdict::Stopped
} else {
WorkerStopVerdict::Refused
}
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, Mutex};
use super::*;
use serde_json::Map;
/// A fake worker: `acks[i]` decides whether the shutdown request
/// succeeds; `exits[i]` whether the process exits in budget. The
/// observed budgets are recorded for the spec-table assertions.
#[derive(Default)]
struct FakeWorkers {
acks: Mutex<Vec<bool>>,
exits: Mutex<Vec<bool>>,
request_budgets_ns: Mutex<Vec<u64>>,
exit_budgets_ns: Mutex<Vec<u64>>,
}
impl WorkerStopTransport for Arc<FakeWorkers> {
fn request_shutdown(
&self,
_resident: &Arc<ResidentWorker>,
timeout: Duration,
) -> impl std::future::Future<Output = Result<()>> + Send {
self.request_budgets_ns
.lock()
.unwrap()
.push(timeout.as_nanos() as u64);
let ack = *self.acks.lock().unwrap().last().unwrap_or(&false);
async move {
if ack {
Ok(())
} else {
Err(anyhow::anyhow!("Session worker timed out"))
}
}
}
fn wait_exit(
&self,
_resident: &Arc<ResidentWorker>,
timeout: Duration,
) -> impl std::future::Future<Output = bool> + Send {
self.exit_budgets_ns
.lock()
.unwrap()
.push(timeout.as_nanos() as u64);
let exit = *self.exits.lock().unwrap().last().unwrap_or(&false);
async move { exit }
}
}
fn resident(name: &str) -> Arc<ResidentWorker> {
// The stop driver only reads `worker_id`; a real descriptor is
// needed because `ResidentWorker::new` takes one.
let descriptor = pa_types::daemon::DaemonWorkerDescriptor {
version: 2,
worker_id: name.to_string(),
pid: 0,
process_start_id: None,
socket_path: "/tmp/w.sock".into(),
recovery_journal_path: "/tmp/w.jsonl".into(),
orphan_process_journal_path: None,
supervisor_socket_path: "/tmp/s.sock".into(),
authentication_token: "t".into(),
worker_instance_id: None,
root_active_session_id: name.to_string(),
owner_client_id: None,
root_session_id: None,
session_file: None,
session_dir: None,
telemetry_disabled: None,
created_at: String::new(),
updated_at: String::new(),
lifecycle: pa_types::daemon::DaemonWorkerLifecycle::Ready,
create_command: pa_types::daemon::DurableDaemonCreateCommand {
session_path: None,
no_session: None,
rest: Map::default(),
},
consecutive_failures: 0,
stop_requested_at: None,
archive_on_stop: None,
last_failure_at: None,
last_error: None,
rest: Map::default(),
};
crate::registry::ResidentWorker::new(
name.to_string(),
descriptor,
std::path::PathBuf::from("/tmp/descriptor.json"),
)
}
fn budget() -> UpdateTimeoutBudget {
UpdateTimeoutBudget {
worker_stop_ms: 30_000,
worker_stop_extension_ms: 30_000,
..UpdateTimeoutBudget::default()
}
}
#[tokio::test]
async fn stopped_and_refused_verdicts_carry_the_spec_budgets() {
let fake = Arc::new(FakeWorkers {
acks: Mutex::new(vec![true, true, false]),
exits: Mutex::new(vec![true, false, false]),
..FakeWorkers::default()
});
// The fake answers per call order; give each worker its own slot by
// driving one resident at a time through the shared driver.
for (expected, ack, exit) in [
(WorkerStopVerdict::Stopped, true, true),
(WorkerStopVerdict::Refused, true, false),
(WorkerStopVerdict::Refused, false, false),
] {
*fake.acks.lock().unwrap() = vec![ack];
*fake.exits.lock().unwrap() = vec![exit];
fake.request_budgets_ns.lock().unwrap().clear();
fake.exit_budgets_ns.lock().unwrap().clear();
let verdicts = stop_workers_gracefully(&fake, &[resident("w")], &budget()).await;
assert_eq!(verdicts.len(), 1);
assert_eq!(verdicts[0].1, expected, "ack={ack} exit={exit}");
assert_eq!(
fake.request_budgets_ns.lock().unwrap().last(),
Some(&30_000_000_000)
);
if ack {
assert_eq!(
fake.exit_budgets_ns.lock().unwrap().last(),
Some(&30_000_000_000)
);
} else {
// A refused request never gets an exit window.
assert_eq!(fake.exit_budgets_ns.lock().unwrap().len(), 0);
}
}
}
#[tokio::test]
async fn no_ack_means_no_kill_and_the_verdict_names_the_worker() {
// The spec's slice-3 verifier: a worker that never acks is Refused
// (the session keeps running — nothing signals or kills).
let fake = Arc::new(FakeWorkers {
acks: Mutex::new(vec![false]),
exits: Mutex::new(vec![false]),
..FakeWorkers::default()
});
let residents = vec![resident("never-acks"), resident("never-acks-2")];
*fake.acks.lock().unwrap() = vec![false, false];
let verdicts = stop_workers_gracefully(&fake, &residents, &budget()).await;
assert_eq!(
verdicts,
vec![
("never-acks".to_string(), WorkerStopVerdict::Refused),
("never-acks-2".to_string(), WorkerStopVerdict::Refused),
]
);
}
#[test]
fn ts_worker_request_timeout_parity() {
// TS `UPDATE_RESTART_WORKER_REQUEST_TIMEOUT_MS = 90_000`.
assert_eq!(WORKER_REQUEST_TIMEOUT_MS, 90_000);
}
}