A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
make on-disk sync and locking match each file's durability class (#3380)
Sebastian Müller97b4a03ffc21
diff --git a/crates/pa-core/src/auth/storage.rs b/crates/pa-core/src/auth/storage.rsindex 4999784931..6db8582eb7 100644--- a/crates/pa-core/src/auth/storage.rs+++ b/crates/pa-core/src/auth/storage.rs@@ -236,7 +236,11 @@ impl AuthStorageBackend for FileAuthStorageBackend {let current = fs::read_to_string(&self.auth_path).ok();let ((), next) = update(current)?;if let Some(next) = next {- super::super::settings::storage::atomic_write(&self.auth_path, &next)?;+ super::super::settings::storage::atomic_write_with(+ &self.auth_path,+ &next,+ super::super::settings::storage::AtomicWriteOptions { fsync: true },+ )?;}drop(guard);Ok(())@@ -307,10 +311,10 @@ mod tests {assert!(data.credential("prime-inference").is_some());}- /// The auth save goes through the real `with_lock` writer and takes NO fsync branch landing the- /// exact document bytes.+ /// The auth save goes through the real `with_lock` writer and takes exactly one fsync branch+ /// landing the exact document bytes.#[test]- fn auth_write_takes_the_ts_default_no_sync() {+ fn auth_write_takes_exactly_one_fsync() {let dir = tempfile::tempdir().unwrap();let backend = FileAuthStorageBackend::new(dir.path().join("auth.json"));let document = r#"{ "prime-inference": { "type": "api_key", "key": "sk" } }"#;@@ -320,8 +324,8 @@ mod tests {.unwrap();assert_eq!(crate::settings::storage::opt_in_fsync_calls(),- before,- "the TS-default auth write must not sync"+ before + 1,+ "the auth write must flush the temp file before the rename");assert_eq!(fs::read_to_string(dir.path().join("auth.json")).unwrap(),diff --git a/crates/pa-core/src/cron/scheduler.rs b/crates/pa-core/src/cron/scheduler.rsindex c82a80b933..acd6cb7a53 100644--- a/crates/pa-core/src/cron/scheduler.rs+++ b/crates/pa-core/src/cron/scheduler.rs@@ -89,7 +89,9 @@ impl<H: AgentCronSchedulerHooks + 'static> AgentCronScheduler<H> {self.core.stopped.store(false, Ordering::SeqCst);if !self.core.has_started.swap(true, Ordering::SeqCst) {let now = self.core.hooks.now();- self.core.store.recover_interrupted_dispatches(now);+ if let Err(error) = self.core.store.recover_interrupted_dispatches(now) {+ tracing::warn!(%error, "cron recovery failed at start");+ }}self.schedule_next().await;}@@ -113,7 +115,7 @@ impl<H: AgentCronSchedulerHooks + 'static> AgentCronScheduler<H> {////// # Errors///- /// The underlying pass never fails in the current implementation.+ /// Returns an error when a cron state lock or write fails.pub async fn run_due(&self) -> anyhow::Result<usize> {self.core.run_due_at(self.core.hooks.now()).await}@@ -135,7 +137,7 @@ impl<H: AgentCronSchedulerHooks + 'static> SchedulerCore<H> {////// # Errors///- /// The current implementation never returns `Err`.+ /// Returns an error when a cron state lock or write fails.pub async fn run_due_at(&self, now: u64) -> anyhow::Result<usize> {// The pass claim is atomic: exactly one pass runs at a time.if (self.stopped.load(Ordering::SeqCst) && self.has_started.load(Ordering::SeqCst))@@ -150,8 +152,8 @@ impl<H: AgentCronSchedulerHooks + 'static> SchedulerCore<H> {let _running = RunningGuard(&self.running);// Recover interrupted dispatches before claiming: a claim left// by an unwound pass is released here.- self.store.recover_interrupted_dispatches(now);- let claimed = self.store.claim_due(now, self.hooks.now());+ self.store.recover_interrupted_dispatches(now)?;+ let claimed = self.store.claim_due(now, self.hooks.now())?;let dispatches: Vec<PendingDispatch> = claimed.into_iter().map(|dispatch| {@@ -203,16 +205,16 @@ impl<H: AgentCronSchedulerHooks + 'static> SchedulerCore<H> {) -> Option<&'static str> {let outcome = async {let Some(job) = self.store.get_claimed_job(&dispatch.job.id) else {- self.store- .record_dispatch_result(- &dispatch.id,- &DispatchResultOptions {- now: Some(self.hooks.now()),- outcome: "skipped",- error: None,- },- )- .ok();+ if let Err(error) = self.store.record_dispatch_result(+ &dispatch.id,+ &DispatchResultOptions {+ now: Some(self.hooks.now()),+ outcome: "skipped",+ error: None,+ },+ ) {+ tracing::warn!(%error, "cron dispatch result not saved");+ }return Some("skipped");};let mut run_error: Option<String> = None;@@ -231,22 +233,25 @@ impl<H: AgentCronSchedulerHooks + 'static> SchedulerCore<H> {"ran"};let failed = run_error.is_some();- self.store- .record_dispatch_result(- &dispatch.id,- &DispatchResultOptions {- now: Some(self.hooks.now()),- outcome,- error: run_error,- },- )- .ok();+ if let Err(error) = self.store.record_dispatch_result(+ &dispatch.id,+ &DispatchResultOptions {+ now: Some(self.hooks.now()),+ outcome,+ error: run_error,+ },+ ) {+ tracing::warn!(%error, "cron dispatch result not saved");+ }if failed {let now = self.hooks.now();let streak = self.note_run_failure(&dispatch.job.id);- let _ = self+ if let Err(error) = self.store- .defer_next_run(&dispatch.job.id, now + failure_backoff_ms(streak));+ .defer_next_run(&dispatch.job.id, now + failure_backoff_ms(streak))+ {+ tracing::warn!(%error, "cron failure backoff not saved");+ }} else if outcome == "ran" {self.clear_run_failure(&dispatch.job.id);}@@ -541,7 +546,10 @@ mod tests {let job = store.create(&input("tick", "every 10m", now)).expect("first job");- store.cancel(&job.id, now).expect("cancel the only job");+ store+ .cancel(&job.id, now)+ .unwrap()+ .expect("cancel the only job");let deadline = std::time::Instant::now() + std::time::Duration::from_millis(300);while std::time::Instant::now() < deadline {tokio::time::sleep(std::time::Duration::from_millis(10)).await;diff --git a/crates/pa-core/src/cron/store/heartbeat.rs b/crates/pa-core/src/cron/store/heartbeat.rsindex 187d4250f3..7c962cb2f7 100644--- a/crates/pa-core/src/cron/store/heartbeat.rs+++ b/crates/pa-core/src/cron/store/heartbeat.rs@@ -101,11 +101,20 @@ impl AgentCronJobStore {};let mut jobs = existing;jobs.push(job.clone());- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;Ok(job)}- pub fn pause_heartbeat(&self, active_session_id: &str, now: u64) -> Option<AgentCronJob> {- let current = self.get_heartbeat(active_session_id)?;+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.+ pub fn pause_heartbeat(+ &self,+ active_session_id: &str,+ now: u64,+ ) -> anyhow::Result<Option<AgentCronJob>> {+ let Some(current) = self.get_heartbeat(active_session_id) else {+ return Ok(None);+ };let now_iso = iso_from_millis(now);let mut paused = None;let jobs: Vec<AgentCronJob> = self@@ -125,8 +134,8 @@ impl AgentCronJobStore {paused_job}).collect();- self.write_jobs(&jobs);- paused+ self.write_jobs(&jobs)?;+ Ok(paused)}/// Resume the heartbeat for a session, recomputing its next run time. `Ok(None)`@@ -165,12 +174,21 @@ impl AgentCronJobStore {resumed_job}).collect();- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;Ok(resumed)}- pub fn clear_heartbeat(&self, active_session_id: &str, now: u64) -> Option<AgentCronJob> {- let current = self.get_heartbeat(active_session_id)?;+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.+ pub fn clear_heartbeat(+ &self,+ active_session_id: &str,+ now: u64,+ ) -> anyhow::Result<Option<AgentCronJob>> {+ let Some(current) = self.get_heartbeat(active_session_id) else {+ return Ok(None);+ };let now_iso = iso_from_millis(now);let mut cleared = None;let jobs: Vec<AgentCronJob> = self@@ -190,8 +208,8 @@ impl AgentCronJobStore {cleared_job}).collect();- self.write_jobs(&jobs);- cleared+ self.write_jobs(&jobs)?;+ Ok(cleared)}/// Apply a pause, stop, or resume management action to a heartbeat job. `Ok(None)`@@ -263,10 +281,10 @@ impl AgentCronJobStore {}}).collect();- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;return Ok(Some(updated_job));}- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;Ok(Some(updated_job))}}@@ -276,6 +294,25 @@ mod tests {use super::*;use crate::cron::store::input;+ #[test]+ fn pause_does_not_ack_when_state_lock_is_held() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("jobs.json");+ let store = AgentCronJobStore::new(path.clone());+ let now = 1_700_000_000_000;+ let heartbeat = store+ .create_heartbeat(&input("continue", "every 5m", now))+ .unwrap();+ let before = std::fs::read(&path).unwrap();+ let held =+ crate::platform::lock_dir::LockDir::acquire(&path, std::time::Duration::from_secs(30))+ .unwrap();+ assert!(store.pause_heartbeat("live-1", now + 1).is_err());+ assert_eq!(std::fs::read(&path).unwrap(), before);+ drop(held);+ assert_eq!(store.get_heartbeat("live-1"), Some(heartbeat));+ }+#[test]fn heartbeat_lifecycle() {let dir = tempfile::TempDir::new().unwrap();@@ -285,7 +322,7 @@ mod tests {.create_heartbeat(&input("continue the mission", "every 5m", now)).unwrap();assert_eq!(store.get_heartbeat("live-1").unwrap().id, heartbeat.id);- let paused = store.pause_heartbeat("live-1", now + 1).unwrap();+ let paused = store.pause_heartbeat("live-1", now + 1).unwrap().unwrap();assert_eq!(paused.status, JobStatus::Paused);assert_eq!(paused.next_run_at, None);let resumed = store.resume_heartbeat("live-1", now + 2).unwrap().unwrap();@@ -299,7 +336,7 @@ mod tests {let jobs = store.list();let cancelled_first = jobs.iter().find(|job| job.id != second.id).unwrap();assert_eq!(cancelled_first.status, JobStatus::Cancelled);- let cleared = store.clear_heartbeat("live-1", now + 4).unwrap();+ let cleared = store.clear_heartbeat("live-1", now + 4).unwrap().unwrap();assert_eq!(cleared.status, JobStatus::Cancelled);assert!(store.create_heartbeat(&input("nope", "in 10m", now))diff --git a/crates/pa-core/src/cron/store/jobs.rs b/crates/pa-core/src/cron/store/jobs.rsindex fce3c0b553..829d04a786 100644--- a/crates/pa-core/src/cron/store/jobs.rs+++ b/crates/pa-core/src/cron/store/jobs.rs@@ -62,13 +62,17 @@ impl AgentCronJobStore {};let mut jobs = self.read_jobs();jobs.push(job.clone());- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;Ok(job)}/// Bind jobs stored for a session file to a live session id on restore, or/// move a live session's jobs to a new file when it switches.- pub fn rebind_session_jobs(&self, input: &SessionBinding) -> Vec<AgentCronJob> {+ ///+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.+ pub fn rebind_session_jobs(&self, input: &SessionBinding) -> anyhow::Result<Vec<AgentCronJob>> {let target_session_file = resolve_path(&input.session_file);let mut rebound_jobs = Vec::new();let jobs: Vec<AgentCronJob> = self@@ -99,11 +103,18 @@ impl AgentCronJobStore {}).collect();if !rebound_jobs.is_empty() {- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;}- rebound_jobs+ Ok(rebound_jobs)}- pub fn cancel_jobs_for_session(&self, input: &CancelJobsFilter, now: u64) -> Vec<AgentCronJob> {+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.+ pub fn cancel_jobs_for_session(+ &self,+ input: &CancelJobsFilter,+ now: u64,+ ) -> anyhow::Result<Vec<AgentCronJob>> {let now_iso = iso_from_millis(now);let target_session_file = input.session_file.as_ref().map(|file| resolve_path(file));let mut cancelled = Vec::new();@@ -136,11 +147,14 @@ impl AgentCronJobStore {}).collect();if !cancelled.is_empty() {- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;}- cancelled+ Ok(cancelled)}- pub fn cancel(&self, id: &str, now: u64) -> Option<AgentCronJob> {+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.+ pub fn cancel(&self, id: &str, now: u64) -> anyhow::Result<Option<AgentCronJob>> {let now_iso = iso_from_millis(now);let mut cancelled = None;let jobs: Vec<AgentCronJob> = self@@ -161,9 +175,9 @@ impl AgentCronJobStore {}).collect();if cancelled.is_some() {- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;}- cancelled+ Ok(cancelled)}/// Record one run: bump counters, roll `nextRunAt`, complete one-shots.@@ -216,13 +230,17 @@ impl AgentCronJobStore {}).collect();if updated.is_some() {- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;}Ok(updated)}/// Record a skipped run: roll `nextRunAt`, stamp `lastSkippedAt`.- pub fn record_skip_result(&self, id: &str, now: u64) -> Option<AgentCronJob> {+ ///+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.+ pub fn record_skip_result(&self, id: &str, now: u64) -> anyhow::Result<Option<AgentCronJob>> {let now_iso = iso_from_millis(now);let mut updated = None;let jobs: Vec<AgentCronJob> = self@@ -247,9 +265,9 @@ impl AgentCronJobStore {}).collect();if updated.is_some() {- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;}- updated+ Ok(updated)}/// Push a job's next run later, never earlier (the scheduler's consecutive-failure backoff); an@@ -258,7 +276,11 @@ impl AgentCronJobStore {/// Runs under the store's state locks: a cancel landing between the failure and this/// defer must stay cancelled, so the status check and the write are one atomic/// state mutation.- pub fn defer_next_run(&self, id: &str, until_ms: u64) -> Option<AgentCronJob> {+ ///+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.+ pub fn defer_next_run(&self, id: &str, until_ms: u64) -> anyhow::Result<Option<AgentCronJob>> {let until_iso = iso_from_millis(until_ms);let mut updated = None;self.mutate_states(|state| {@@ -282,8 +304,8 @@ impl AgentCronJobStore {}).collect();Vec::new()- });- updated+ })?;+ Ok(updated)}pub fn due(&self, now: u64) -> Vec<AgentCronJob> {@@ -294,7 +316,15 @@ impl AgentCronJobStore {}/// Atomically claim due jobs: advance their schedule and record dispatches.- pub fn claim_due(&self, due_at: u64, claimed_at: u64) -> Vec<AgentCronDispatch> {+ ///+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.+ pub fn claim_due(+ &self,+ due_at: u64,+ claimed_at: u64,+ ) -> anyhow::Result<Vec<AgentCronDispatch>> {self.mutate_states(|state| claim_due_in_state(state, due_at, claimed_at))}@@ -320,7 +350,7 @@ impl AgentCronJobStore {////// # Errors///- /// The current implementation never returns `Err`.+ /// Returns an error when the state lock or write fails.pub fn record_dispatch_result(&self,dispatch_id: &str,@@ -377,32 +407,38 @@ impl AgentCronJobStore {}).collect();Vec::new()- });+ })?;Ok(updated)}- pub fn recover_interrupted_dispatches(&self, now: u64) -> Vec<AgentCronJob> {+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.+ pub fn recover_interrupted_dispatches(&self, now: u64) -> anyhow::Result<Vec<AgentCronJob>> {let mut recovered = Vec::new();self.mutate_states(|state| {recover_interrupted_in_state(state, now, &mut recovered, None);Vec::new()- });- recovered+ })?;+ Ok(recovered)}+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.pub fn recover_interrupted_dispatches_by_id(&self,dispatch_ids: &[String],now: u64,- ) -> Vec<AgentCronJob> {+ ) -> anyhow::Result<Vec<AgentCronJob>> {let mut recovered = Vec::new();let dispatch_ids: std::collections::HashSet<String> =dispatch_ids.iter().cloned().collect();self.mutate_states(|state| {recover_interrupted_in_state(state, now, &mut recovered, Some(&dispatch_ids));Vec::new()- });- recovered+ })?;+ Ok(recovered)}pub fn get_due_job(&self, id: &str, now: u64) -> Option<AgentCronJob> {@@ -439,20 +475,37 @@ mod tests {assert_eq!(store.list().len(), 1);let due = store.due(now + 600_000);assert!(due.iter().any(|found| found.id == job.id));- let cancelled = store.cancel(&job.id, now + 1).unwrap();+ let cancelled = store.cancel(&job.id, now + 1).unwrap().unwrap();assert_eq!(cancelled.status, JobStatus::Cancelled);assert_eq!(store.list()[0].status, JobStatus::Cancelled);assert!(store.create(&input(" ", "every 10m", now)).is_err());}+ #[test]+ fn cancel_does_not_ack_when_state_lock_is_held() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("jobs.json");+ let store = AgentCronJobStore::new(path.clone());+ let now = 1_700_000_000_000;+ let job = store.create(&input("tick", "every 10m", now)).unwrap();+ let before = std::fs::read(&path).unwrap();+ let held =+ crate::platform::lock_dir::LockDir::acquire(&path, std::time::Duration::from_secs(30))+ .unwrap();+ assert!(store.cancel(&job.id, now + 1).is_err());+ assert_eq!(std::fs::read(&path).unwrap(), before);+ drop(held);+ assert_eq!(store.list(), vec![job]);+ }+#[test]fn claim_dispatch_and_result_recording() {let dir = tempfile::TempDir::new().unwrap();let store = AgentCronJobStore::new(dir.path().join("jobs.json"));let now = 1_700_000_000_000;let job = store.create(&input("tick", "every 10m", now)).unwrap();- assert!(store.claim_due(now, now).is_empty());- let dispatches = store.claim_due(now + 600_000, now + 600_000);+ assert!(store.claim_due(now, now).unwrap().is_empty());+ let dispatches = store.claim_due(now + 600_000, now + 600_000).unwrap();assert_eq!(dispatches.len(), 1);assert_eq!(dispatches[0].job.id, job.id);assert_eq!(@@ -473,9 +526,11 @@ mod tests {.unwrap();assert_eq!(updated.run_count, 1);assert!(store.get_claimed_job(&job.id).is_none());- let second = store.claim_due(now + 1_200_000, now + 1_200_000);+ let second = store.claim_due(now + 1_200_000, now + 1_200_000).unwrap();assert_eq!(second.len(), 1);- let recovered = store.recover_interrupted_dispatches(now + 1_300_000);+ let recovered = store+ .recover_interrupted_dispatches(now + 1_300_000)+ .unwrap();assert_eq!(recovered.len(), 1);assert_eq!(recovered[0].last_error.as_deref(),@@ -522,6 +577,28 @@ mod tests {assert!(store.list().iter().any(|job| job.last_skipped_at.is_some()));}+ #[test]+ fn claim_due_skips_while_the_state_lock_is_held() {+ let dir = tempfile::tempdir().unwrap();+ let store = AgentCronJobStore::new(dir.path().join("jobs.json"));+ let now = 1_700_000_000_000;+ let job = store.create(&input("tick", "every 10m", now)).unwrap();+ let path = store.require_file_path();+ let before = std::fs::read(&path).unwrap();+ {+ let _held = crate::platform::lock_dir::LockDir::acquire(+ &path,+ std::time::Duration::from_secs(30),+ )+ .unwrap();+ assert!(store.claim_due(now + 600_000, now + 600_000).is_err());+ assert_eq!(std::fs::read(&path).unwrap(), before);+ }+ let dispatches = store.claim_due(now + 600_000, now + 600_000).unwrap();+ assert_eq!(dispatches.len(), 1);+ assert_eq!(dispatches[0].job.id, job.id);+ }+#[test]fn defer_next_run_pushes_only_later() {let dir = tempfile::TempDir::new().unwrap();@@ -529,19 +606,28 @@ mod tests {let now = 1_700_000_000_000;let job = store.create(&input("tick", "every 10m", now)).unwrap();let scheduled_next = now + 600_000;- assert!(store.defer_next_run(&job.id, now + 60_000).is_none());+ assert!(store+ .defer_next_run(&job.id, now + 60_000)+ .unwrap()+ .is_none());let current = store.list().pop().expect("job kept");assert_eq!(crate::cron::parse_iso_millis(current.next_run_at.as_deref().unwrap()),Some(scheduled_next));let deferred = now + 900_000;- let updated = store.defer_next_run(&job.id, deferred).expect("deferred");+ let updated = store+ .defer_next_run(&job.id, deferred)+ .unwrap()+ .expect("deferred");assert_eq!(crate::cron::parse_iso_millis(updated.next_run_at.as_deref().unwrap()),Some(deferred));- assert!(store.defer_next_run(&job.id, now + 120_000).is_none());+ assert!(store+ .defer_next_run(&job.id, now + 120_000)+ .unwrap()+ .is_none());let current = store.list().pop().expect("job kept");assert_eq!(crate::cron::parse_iso_millis(current.next_run_at.as_deref().unwrap()),@@ -550,6 +636,9 @@ mod tests {// Cancelled jobs are never deferred. Fresh clock: the jobs file merges on// `updatedAt` freshness, so a stale stamp would lose the cancel.store.cancel(&job.id, now_millis()).unwrap();- assert!(store.defer_next_run(&job.id, now + 2_000_000).is_none());+ assert!(store+ .defer_next_run(&job.id, now + 2_000_000)+ .unwrap()+ .is_none());}}diff --git a/crates/pa-core/src/cron/store/rlm_heartbeat.rs b/crates/pa-core/src/cron/store/rlm_heartbeat.rsindex 565c9d1be5..912475f123 100644--- a/crates/pa-core/src/cron/store/rlm_heartbeat.rs+++ b/crates/pa-core/src/cron/store/rlm_heartbeat.rs@@ -87,7 +87,7 @@ impl AgentCronJobStore {};let mut jobs = self.read_jobs();jobs.push(job.clone());- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;Ok(job)}@@ -169,7 +169,7 @@ impl AgentCronJobStore {.collect();if matched {if let Some(updated_job) = &updated {- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;return Ok(Some(updated_job.clone()));}anyhow::bail!(@@ -179,12 +179,15 @@ impl AgentCronJobStore {Ok(None)}+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.pub fn delete_rlm_heartbeat(&self,active_session_id: &str,id: &str,now: u64,- ) -> Option<AgentCronJob> {+ ) -> anyhow::Result<Option<AgentCronJob>> {let now_iso = iso_from_millis(now);let mut deleted = None;let jobs: Vec<AgentCronJob> = self@@ -208,16 +211,19 @@ impl AgentCronJobStore {}).collect();if deleted.is_some() {- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;}- deleted+ Ok(deleted)}+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.pub fn cancel_rlm_heartbeats_for_session(&self,active_session_id: &str,now: u64,- ) -> Vec<AgentCronJob> {+ ) -> anyhow::Result<Vec<AgentCronJob>> {let now_iso = iso_from_millis(now);let mut cancelled = Vec::new();let jobs: Vec<AgentCronJob> = self@@ -241,9 +247,9 @@ impl AgentCronJobStore {}).collect();if !cancelled.is_empty() {- self.write_jobs(&jobs);+ self.write_jobs(&jobs)?;}- cancelled+ Ok(cancelled)}}@@ -289,10 +295,13 @@ mod tests {assert_eq!(resumed.status, JobStatus::Active);let deleted = store.delete_rlm_heartbeat("live-1", &rlm.id, now + 3)+ .unwrap().unwrap();assert_eq!(deleted.status, JobStatus::Cancelled);let second = store.create_rlm_heartbeat(&rlm_input).unwrap();- let cancelled = store.cancel_rlm_heartbeats_for_session("live-1", now + 4);+ let cancelled = store+ .cancel_rlm_heartbeats_for_session("live-1", now + 4)+ .unwrap();assert_eq!(cancelled.len(), 1);assert_eq!(second.status, JobStatus::Active);assert_eq!(cancelled[0].status, JobStatus::Cancelled);diff --git a/crates/pa-core/src/cron/store/session_artifacts.rs b/crates/pa-core/src/cron/store/session_artifacts.rsindex df6df041d2..8ce5e1273f 100644--- a/crates/pa-core/src/cron/store/session_artifacts.rs+++ b/crates/pa-core/src/cron/store/session_artifacts.rs@@ -42,26 +42,37 @@ impl AgentCronJobStore {files.clone().into_iter().collect()}- pub fn recover_session_artifact(&self, session_id: &str, now: u64) -> Vec<AgentCronJob> {+ /// # Errors+ ///+ /// Returns an error when the state lock or write fails.+ pub fn recover_session_artifact(+ &self,+ session_id: &str,+ now: u64,+ ) -> anyhow::Result<Vec<AgentCronJob>> {let Some(path) = self.session_artifacts().into_iter().find(|(id, _)| id == session_id).map(|(_, path)| path)else {- return Vec::new();+ return Ok(Vec::new());};- with_state_locks(std::slice::from_ref(&path), || {- let mut state = read_jobs_state(&path);- let mut recovered = Vec::new();- if !state.dispatches.is_empty() {- recover_interrupted_in_state(&mut state, now, &mut recovered, None);- write_jobs_state(&path, &state);- }- recovered- })+ with_state_locks(+ std::slice::from_ref(&path),+ || -> anyhow::Result<Vec<AgentCronJob>> {+ let mut state = read_jobs_state(&path);+ let mut recovered = Vec::new();+ if !state.dispatches.is_empty() {+ recover_interrupted_in_state(&mut state, now, &mut recovered, None);+ write_jobs_state(&path, &state)?;+ }+ Ok(recovered)+ },+ )+ .ok_or_else(|| anyhow::anyhow!("cron jobs state lock not acquired; skipped"))?}- pub(crate) fn write_jobs_session_artifacts(&self, jobs: &[AgentCronJob]) {+ pub(crate) fn write_jobs_session_artifacts(&self, jobs: &[AgentCronJob]) -> anyhow::Result<()> {let artifact_files = self.session_artifact_files.lock()@@ -71,12 +82,14 @@ impl AgentCronJobStore {artifact_files.keys().cloned().collect();for job in jobs {if !registered.contains(&job.session_id) {- // Mirror the TS error contract.- return;+ anyhow::bail!(+ "cron job session artifact not registered: {}",+ job.session_id+ );}}let paths: Vec<PathBuf> = artifact_files.values().cloned().collect();- with_state_locks(&paths, || {+ let with_locks = with_state_locks(&paths, || -> anyhow::Result<()> {let current_by_session_id: HashMap<String, CronJobsState> = artifact_files.iter().map(|(session_id, path)| (session_id.clone(), read_jobs_state(path)))@@ -136,18 +149,20 @@ impl AgentCronJobStore {if serde_json::to_string(¤t).unwrap_or_default()!= serde_json::to_string(&next_state).unwrap_or_default(){- write_jobs_state(path, &next_state);+ write_jobs_state(path, &next_state)?;}}+ Ok(())});+ with_locks.ok_or_else(|| anyhow::anyhow!("cron jobs state lock not acquired; skipped"))?}}-/// Read one scheduled-jobs artifact file's job rows: a locked, read-only scan A missing or+/// Read one scheduled-jobs artifact file's job rows. A missing or/// unparseable file reads as no jobs.#[must_use]pub fn read_scheduled_jobs_artifact(path: &Path) -> Vec<AgentCronJob> {- with_state_locks(&[path.to_path_buf()], || read_jobs_state(path).jobs)+ read_jobs_state(path).jobs}#[cfg(test)]@@ -178,21 +193,25 @@ mod tests {assert_eq!(state_a.jobs[0].prompt, "job a");let state_b = read_jobs_state(&artifacts_b.join(SESSION_SCHEDULED_JOBS_FILENAME));assert_eq!(state_b.jobs.len(), 1);- let rebound = store.rebind_session_jobs(&SessionBinding {- active_session_id: "live-2".to_string(),- session_id: "session-1".to_string(),- session_file: "/w/session.jsonl".to_string(),- cwd: "/w".to_string(),- });+ let rebound = store+ .rebind_session_jobs(&SessionBinding {+ active_session_id: "live-2".to_string(),+ session_id: "session-1".to_string(),+ session_file: "/w/session.jsonl".to_string(),+ cwd: "/w".to_string(),+ })+ .unwrap();assert_eq!(rebound.len(), 2);assert!(rebound.iter().all(|job| job.active_session_id == "live-2"));- let cancelled = store.cancel_jobs_for_session(- &CancelJobsFilter {- session_file: Some("/w/session.jsonl".to_string()),- ..Default::default()- },- now + 1,- );+ let cancelled = store+ .cancel_jobs_for_session(+ &CancelJobsFilter {+ session_file: Some("/w/session.jsonl".to_string()),+ ..Default::default()+ },+ now + 1,+ )+ .unwrap();// Both jobs share the session file (the rebind moved both), so both cancel.assert_eq!(cancelled.len(), 2);}diff --git a/crates/pa-core/src/cron/store/state.rs b/crates/pa-core/src/cron/store/state.rsindex c0e258fb97..30e85e71bb 100644--- a/crates/pa-core/src/cron/store/state.rs+++ b/crates/pa-core/src/cron/store/state.rs@@ -39,7 +39,7 @@ impl AgentCronJobStore {pub(crate) fn mutate_states(&self,mut mutator: impl FnMut(&mut CronJobsState) -> Vec<AgentCronDispatch>,- ) -> Vec<AgentCronDispatch> {+ ) -> anyhow::Result<Vec<AgentCronDispatch>> {let paths: Vec<PathBuf> = if self.session_artifact_mode {self.session_artifact_files.lock()@@ -59,31 +59,34 @@ impl AgentCronJobStore {let before = serde_json::to_string(&state).unwrap_or_default();dispatches.extend(mutator(&mut state));if serde_json::to_string(&state).unwrap_or_default() != before {- write_jobs_state(path, &state);+ write_jobs_state(path, &state)?;changed = true;}}- dispatches- });+ Ok::<_, anyhow::Error>(dispatches)+ })+ .ok_or_else(|| anyhow::anyhow!("cron jobs state lock not acquired; skipped"))??;if changed && Self::heartbeat_catalog_signature(&self.read_jobs()) != previous_heartbeats {self.notify_heartbeat_change();}- dispatches+ Ok(dispatches)}- pub(crate) fn write_jobs(&self, jobs: &[AgentCronJob]) {+ pub(crate) fn write_jobs(&self, jobs: &[AgentCronJob]) -> anyhow::Result<()> {let previous_heartbeats = Self::heartbeat_catalog_signature(&self.read_jobs());if self.session_artifact_mode {- self.write_jobs_session_artifacts(jobs);+ self.write_jobs_session_artifacts(jobs)?;} else {let path = self.require_file_path();with_state_locks(std::slice::from_ref(&path), || {- write_jobs_file(&path, jobs, true);- });+ write_jobs_file(&path, jobs, true)+ })+ .ok_or_else(|| anyhow::anyhow!("cron jobs state lock not acquired; skipped"))??;}if Self::heartbeat_catalog_signature(&self.read_jobs()) != previous_heartbeats {self.notify_heartbeat_change();}+ Ok(())}pub(crate) fn require_file_path(&self) -> PathBuf {@@ -257,7 +260,7 @@ pub(crate) fn recover_interrupted_in_state(}/// Cross-process state locks: lockfile with stale takeover, sorted by path.-pub(crate) fn with_state_locks<T>(paths: &[PathBuf], action: impl FnOnce() -> T) -> T {+pub(crate) fn with_state_locks<T>(paths: &[PathBuf], action: impl FnOnce() -> T) -> Option<T> {let mut unique: Vec<PathBuf> = paths.to_vec();unique.sort();unique.dedup();@@ -269,41 +272,26 @@ pub(crate) fn with_state_locks<T>(paths: &[PathBuf], action: impl FnOnce() -> T)// TS `withCronJobsStateLocks`: proper-lockfile, 100 attempts x// 10ms, 30s staleness takeover.let stale = std::time::Duration::from_millis(LOCK_STALE_MS);- let mut acquired = false;- let mut failure: Option<std::io::Error> = None;- for _ in 0..100 {- match crate::platform::lock_dir::LockDir::acquire(path, stale) {- Ok(guard) => {- guards.push(guard);- acquired = true;- break;- }- Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {- std::thread::sleep(std::time::Duration::from_millis(10));- }- Err(error) => {- failure = Some(error);- break;- }+ match crate::platform::lock_dir::LockDir::acquire_retrying(+ path,+ stale,+ 100,+ std::time::Duration::from_millis(10),+ ) {+ Ok(guard) => guards.push(guard),+ Err(error) => {+ tracing::warn!(+ error = %error,+ path = %path.display(),+ "cron jobs state lock not acquired; skipped"+ );+ return None;}}- if !acquired {- // TS throws when the lock is not acquired; the action still runs, but never silently: a- // concurrent writer may be mutating the file.- tracing::warn!(- error = failure.as_ref().map_or_else(- || "lock still held after retries".to_string(),- ToString::to_string,- ),- path = %path.display(),- "cron jobs state lock not acquired; running unlocked"- );- break;- }}let result = action();drop(guards);- result+ Some(result)}pub(crate) fn read_jobs_state(path: &Path) -> CronJobsState {@@ -338,7 +326,7 @@ pub(crate) fn read_jobs_state(path: &Path) -> CronJobsState {}}-fn write_jobs_file(path: &Path, jobs: &[AgentCronJob], merge_current: bool) {+fn write_jobs_file(path: &Path, jobs: &[AgentCronJob], merge_current: bool) -> anyhow::Result<()> {let current = read_jobs_state(path);let jobs = if merge_current {merge_fresh_jobs(current.jobs, jobs.to_vec())@@ -351,27 +339,35 @@ fn write_jobs_file(path: &Path, jobs: &[AgentCronJob], merge_current: bool) {jobs,dispatches: current.dispatches,},- );+ )}-pub(crate) fn write_jobs_state(path: &Path, state: &CronJobsState) {+pub(crate) fn write_jobs_state(path: &Path, state: &CronJobsState) -> anyhow::Result<()> {if let Some(parent) = path.parent() {- let _ = std::fs::create_dir_all(parent);+ std::fs::create_dir_all(parent)?;}- let serialized = serde_json::to_string_pretty(state).unwrap_or_default();- // The one opt-in fsync: losing the atomic rename rolls back to the previous valid file, which- // cron recovery tolerates (TS `writeJobsState` passes `{ mode: 0o600, fsync: true }`).- let _ = crate::settings::storage::atomic_write_with(+ let serialized = serde_json::to_string_pretty(state)?;+ // Cron state opts into the shared durable write (TS `writeJobsState`+ // passes `{ mode: 0o600, fsync: true }`).+ crate::settings::storage::atomic_write_with(path,&format!("{serialized}\n"),crate::settings::storage::AtomicWriteOptions { fsync: true },- );+ )}#[cfg(test)]mod tests {use super::*;+ #[test]+ fn jobs_state_write_propagates_physical_failure() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("jobs.json");+ std::fs::create_dir(&path).unwrap();+ assert!(write_jobs_state(&path, &CronJobsState::default()).is_err());+ }+/// The cron state write opts in (the TS test pins `{ fsync: true, mode: 0o600 }`)/// `to_string_pretty(state) + "\n"` bytes.#[test]@@ -381,7 +377,7 @@ mod tests {let state = CronJobsState::default();let expected = format!("{}\n", serde_json::to_string_pretty(&state).unwrap());let before = crate::settings::storage::opt_in_fsync_calls();- write_jobs_state(&path, &state);+ write_jobs_state(&path, &state).unwrap();assert_eq!(crate::settings::storage::opt_in_fsync_calls(),before + 1,diff --git a/crates/pa-core/src/mcp/connection_store.rs b/crates/pa-core/src/mcp/connection_store.rsindex f348a5800f..21f01cc3e9 100644--- a/crates/pa-core/src/mcp/connection_store.rs+++ b/crates/pa-core/src/mcp/connection_store.rs@@ -9,7 +9,7 @@ use std::time::{SystemTime, UNIX_EPOCH};use serde::{Deserialize, Serialize};-use crate::settings::storage::atomic_write;+use crate::settings::storage::{atomic_write_with, AtomicWriteOptions};/// The connections file version.const FILE_VERSION: u8 = 1;@@ -231,7 +231,11 @@ fn write_records("version": FILE_VERSION,"connections": ordered,});- atomic_write(path, &serde_json::to_string_pretty(&doc)?)?;+ atomic_write_with(+ path,+ &serde_json::to_string_pretty(&doc)?,+ AtomicWriteOptions { fsync: true },+ )?;Ok(())}@@ -262,11 +266,10 @@ pub fn new_pending_record(mod tests {use super::*;- /// Per-call-site served-path oracle (the TS writer passes only `{ mode: 0o600 }`): the- /// registry write goes through the real `write_records` writer and takes NO fsync- /// branch.+ /// Per-call-site served-path oracle: the registry write goes through the real+ /// `write_records` writer and takes exactly one fsync branch.#[test]- fn registry_write_takes_the_ts_default_no_sync() {+ fn registry_write_takes_exactly_one_fsync() {let dir = tempfile::tempdir().unwrap();let path = dir.path().join("connections.json");let record = McpConnectionRecord {@@ -293,8 +296,8 @@ mod tests {write_records(&path, &records).unwrap();assert_eq!(crate::settings::storage::opt_in_fsync_calls(),- before,- "the TS-default registry write must not sync"+ before + 1,+ "the registry write must flush the temp file before the rename");assert_eq!(std::fs::read_to_string(&path).unwrap(), expected);}diff --git a/crates/pa-core/src/platform/fs.rs b/crates/pa-core/src/platform/fs.rsindex 9ee89da818..1c71628697 100644--- a/crates/pa-core/src/platform/fs.rs+++ b/crates/pa-core/src/platform/fs.rs@@ -2,6 +2,27 @@use std::fs::File;use std::io;+use std::path::Path;++/// fsync a directory so a completed rename survives a crash on POSIX.+///+/// # Errors+///+/// Returns an error when the directory cannot be opened or synced.+#[cfg(unix)]+pub fn sync_directory(path: &Path) -> io::Result<()> {+ File::open(path)?.sync_all()+}++/// Directory fsync is not exposed by the standard library on this platform.+///+/// # Errors+///+/// This implementation never fails.+#[cfg(not(unix))]+pub fn sync_directory(_path: &Path) -> io::Result<()> {+ Ok(())+}/// Flush one file's written bytes with a plain `fsync(2)`; std's/// `sync_data`/`sync_all` take the `F_FULLFSYNC` barrier on Apple.diff --git a/crates/pa-core/src/platform/lock_dir.rs b/crates/pa-core/src/platform/lock_dir.rsindex 5cfcd4c872..15c0977a83 100644--- a/crates/pa-core/src/platform/lock_dir.rs+++ b/crates/pa-core/src/platform/lock_dir.rs@@ -1,7 +1,6 @@//! Cross-process directory locks, byte-compatible with the TS product's-//! `proper-lockfile` 4.1.2 convention: a lock is an EMPTY DIRECTORY at-//! `{file}.lock`, staleness is judged from its bumped mtime alone (no pid-//! or owner file), and a regular file at the lock path is removed on acquisition.+//! `proper-lockfile` 4.1.2 convention: ordinary locks are empty directories+//! at `{file}.lock`; harness-state locks add an owner file for safe stale reclaim.use std::fs;use std::io;@@ -171,6 +170,7 @@ mod win32 {#[derive(Debug)]pub struct LockDir {path: PathBuf,+ owner: Option<String>,}impl LockDir {@@ -191,18 +191,26 @@ impl LockDir {////// [`io::ErrorKind::WouldBlock`] for a fresh foreign lock; other I/O errors as-is.pub fn acquire(file: &Path, stale_after: Duration) -> io::Result<Self> {+ Self::acquire_with_owner(file, stale_after, None)+ }++ fn acquire_with_owner(+ file: &Path,+ stale_after: Duration,+ owner: Option<String>,+ ) -> io::Result<Self> {let path = Self::path_for(file);let stale_after = stale_after.max(MIN_STALE);- match Self::create(&path) {- Ok(()) => Ok(LockDir { path }),+ match Self::create(&path, owner.as_deref()) {+ Ok(()) => Ok(LockDir { path, owner }),// Only an existing path is a lock collision; other failures// (missing parent, permissions) are real errors, never contention.Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {Self::judge_and_reclaim(&path, stale_after)?;// The judge path removed (or raced away) the incumbent: one// fresh attempt; a reappearing rival is contention.- match Self::create(&path) {- Ok(()) => Ok(LockDir { path }),+ match Self::create(&path, owner.as_deref()) {+ Ok(()) => Ok(LockDir { path, owner }),Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {Err(io::Error::new(io::ErrorKind::WouldBlock,@@ -216,38 +224,158 @@ impl LockDir {}}- /// The mkdir is the acquisition signal: EEXIST is the only collision.- #[cfg(unix)]- fn create(path: &Path) -> io::Result<()> {- fs::create_dir(path)?;- let (sec, nanos) = probe_mtime();- if let Err(error) = set_mtime(path, sec, nanos) {- // Never leave a lock artifact behind a failed probe.- let _ = fs::remove_dir(path);- return Err(error);+ /// [`Self::acquire`] with a bounded retry: only a fresh foreign lock+ /// (`WouldBlock`) is retried, sleeping `interval` between attempts;+ /// any other error returns immediately, and the final `WouldBlock`+ /// is returned after the last attempt.+ ///+ /// # Errors+ ///+ /// The last [`io::ErrorKind::WouldBlock`] when all attempts contend;+ /// other I/O errors as-is.+ pub fn acquire_retrying(+ file: &Path,+ stale_after: Duration,+ attempts: u32,+ interval: Duration,+ ) -> io::Result<Self> {+ let mut attempt = 0;+ loop {+ match Self::acquire(file, stale_after) {+ Ok(guard) => return Ok(guard),+ Err(error) if error.kind() != io::ErrorKind::WouldBlock => return Err(error),+ Err(error) => {+ attempt += 1;+ if attempt >= attempts {+ return Err(error);+ }+ std::thread::sleep(interval);+ }+ }+ }+ }++ /// Acquire a harness-state lock with a PID and per-process token.+ /// Only a provably dead owner can be reclaimed after `stale_after`.+ ///+ /// # Errors+ ///+ /// Returns lock contention or an I/O error from acquisition.+ pub fn acquire_owned_retrying(+ file: &Path,+ stale_after: Duration,+ attempts: u32,+ interval: Duration,+ ) -> io::Result<Self> {+ static PROCESS_TOKEN: std::sync::OnceLock<uuid::Uuid> = std::sync::OnceLock::new();+ let token = PROCESS_TOKEN.get_or_init(uuid::Uuid::new_v4);+ let owner = format!("{} {token}.{}", std::process::id(), uuid::Uuid::new_v4());+ let mut attempt = 0;+ loop {+ match Self::acquire_with_owner(file, stale_after, Some(owner.clone())) {+ Ok(guard) => return Ok(guard),+ Err(error) if error.kind() != io::ErrorKind::WouldBlock => return Err(error),+ Err(error) => {+ attempt += 1;+ if attempt >= attempts {+ return Err(error);+ }+ std::thread::sleep(interval);+ }+ }+ }+ }++ /// Check that an owned lock is still held before writing its state file.+ ///+ /// # Errors+ ///+ /// Returns an error if the owner file no longer matches this guard.+ pub fn ensure_owned(&self) -> io::Result<()> {+ if self+ .owner+ .as_ref()+ .is_some_and(|owner| !Self::owner_matches(&self.path, owner))+ {+ return Err(io::Error::other(format!(+ "harness state lock lost: {}",+ self.path.display()+ )));}Ok(())}- /// The mkdir is the acquisition signal; the mtime probe keeps staleness- /// meaningful on NTFS (directory mtimes otherwise sit on the second).- #[cfg(windows)]- fn create(path: &Path) -> io::Result<()> {+ fn owner_matches(path: &Path, owner: &str) -> bool {+ fs::read_to_string(path.join("owner")).is_ok_and(|recorded| recorded.trim() == owner)+ }++ fn owner_dead(recorded: &str) -> bool {+ let Some((pid, token)) = recorded.trim().split_once(' ') else {+ return true;+ };+ if token.is_empty() {+ return true;+ }+ let Ok(pid) = pid.parse::<u32>() else {+ return true;+ };+ if pid == 0 || pid > i32::MAX as u32 {+ return true;+ }+ #[cfg(unix)]+ {+ let result = unsafe { libc::kill(pid as i32, 0) };+ result != 0 && io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH)+ }+ #[cfg(windows)]+ {+ matches!(+ pa_types::platform::process::is_process_alive(pid),+ Ok(false)+ )+ }+ #[cfg(not(any(unix, windows)))]+ {+ false+ }+ }++ /// The mkdir is the acquisition signal: EEXIST is the only collision.+ #[cfg(any(unix, windows))]+ fn create(path: &Path, owner: Option<&str>) -> io::Result<()> {+ #[cfg(unix)]+ use std::os::unix::fs::PermissionsExt;fs::create_dir(path)?;- let (sec, nanos) = probe_mtime();- if let Err(error) = set_mtime(path, sec, nanos) {- // Never leave a lock artifact behind a failed probe.+ let result = (|| {+ if let Some(owner) = owner {+ #[cfg(unix)]+ {+ fs::set_permissions(path, fs::Permissions::from_mode(0o700))?;+ }+ fs::write(path.join("owner"), format!("{owner}\n"))?;+ #[cfg(unix)]+ fs::set_permissions(path.join("owner"), fs::Permissions::from_mode(0o600))?;+ }+ let (sec, nanos) = probe_mtime();+ set_mtime(path, sec, nanos)+ })();+ if result.is_err() {+ let _ = fs::remove_file(path.join("owner"));let _ = fs::remove_dir(path);- return Err(error);}- Ok(())+ result}#[cfg(not(any(unix, windows)))]- fn create(path: &Path) -> io::Result<()> {- // No mtime probe on this platform: staleness is judged from the- // filesystem's own directory mtime.- fs::create_dir(path)+ fn create(path: &Path, owner: Option<&str>) -> io::Result<()> {+ fs::create_dir(path)?;+ if let Some(owner) = owner {+ if let Err(error) = fs::write(path.join("owner"), format!("{owner}\n")) {+ let _ = fs::remove_dir(path);+ return Err(error);+ }+ }+ Ok(())}/// Decide the fate of an incumbent at `path`. Returns only when the@@ -286,6 +414,22 @@ impl LockDir {.duration_since(modified).unwrap_or_default();if age > stale_after {+ let owner_path = path.join("owner");+ let recorded = fs::read_to_string(&owner_path).ok();+ if recorded+ .as_deref()+ .is_some_and(|owner| !Self::owner_dead(owner))+ {+ return Err(io::Error::new(+ io::ErrorKind::WouldBlock,+ format!("Lock file is already being held: {}", path.display()),+ ));+ }+ match fs::remove_file(&owner_path) {+ Ok(()) => {}+ Err(error) if error.kind() == io::ErrorKind::NotFound => {}+ Err(error) => return Err(error),+ }// Stale: remove and let the caller retry.match fs::remove_dir(path) {Ok(()) => return Ok(()),@@ -319,6 +463,15 @@ impl LockDir {/// else already reclaimed it (the TS release tolerates ENOENT); other/// failures go to the trace log (`Drop` cannot propagate).pub fn release(&self) {+ if let Some(owner) = &self.owner {+ if !Self::owner_matches(&self.path, owner) {+ return;+ }+ if let Err(error) = fs::remove_file(self.path.join("owner")) {+ tracing::warn!("failed to release lock {}: {error}", self.path.display());+ return;+ }+ }if let Err(error) = fs::remove_dir(&self.path) {if error.kind() != io::ErrorKind::NotFound {tracing::warn!("failed to release lock {}: {error}", self.path.display());@@ -405,6 +558,55 @@ mod tests {assert!(!stale.exists());}+ #[cfg(unix)]+ #[test]+ fn live_owned_lock_cannot_be_stolen_after_stale_age() {+ let dir = tempfile::tempdir().unwrap();+ let file = dir.path().join("harness_state.json");+ let guard =+ LockDir::acquire_owned_retrying(&file, Duration::from_secs(10), 1, MIN_STALE).unwrap();+ set_mtime(&guard.path, 1, 0).unwrap();+ let error = LockDir::acquire_owned_retrying(&file, Duration::from_secs(10), 1, MIN_STALE)+ .unwrap_err();+ assert_eq!(error.kind(), io::ErrorKind::WouldBlock);+ guard.ensure_owned().unwrap();+ }++ #[cfg(unix)]+ #[test]+ fn dead_owned_lock_is_reclaimed_and_old_guard_cannot_remove_successor() {+ let dir = tempfile::tempdir().unwrap();+ let file = dir.path().join("harness_state.json");+ let old =+ LockDir::acquire_owned_retrying(&file, Duration::from_secs(10), 1, MIN_STALE).unwrap();+ let mut child = std::process::Command::new("true").spawn().unwrap();+ let dead_pid = child.id();+ child.wait().unwrap();+ fs::write(old.path.join("owner"), format!("{dead_pid} dead-token\n")).unwrap();+ set_mtime(&old.path, 1, 0).unwrap();+ let next =+ LockDir::acquire_owned_retrying(&file, Duration::from_secs(10), 1, MIN_STALE).unwrap();+ assert!(old.ensure_owned().is_err());+ drop(old);+ next.ensure_owned().unwrap();+ drop(next);+ assert!(!LockDir::path_for(&file).exists());+ }++ #[cfg(unix)]+ #[test]+ fn unparseable_owned_lock_is_reclaimed_by_age() {+ let dir = tempfile::tempdir().unwrap();+ let file = dir.path().join("harness_state.json");+ let path = LockDir::path_for(&file);+ fs::create_dir(&path).unwrap();+ fs::write(path.join("owner"), [0xff]).unwrap();+ set_mtime(&path, 1, 0).unwrap();+ let next =+ LockDir::acquire_owned_retrying(&file, Duration::from_secs(10), 1, MIN_STALE).unwrap();+ next.ensure_owned().unwrap();+ }+#[test]fn legacy_lock_file_is_removed_not_choked_on() {// A pre-compat Rust build left flock FILES at the lock path (the TSdiff --git a/crates/pa-core/src/refinement/mod.rs b/crates/pa-core/src/refinement/mod.rsindex 025807fb04..309a1f00b7 100644--- a/crates/pa-core/src/refinement/mod.rs+++ b/crates/pa-core/src/refinement/mod.rs@@ -283,10 +283,40 @@ pub fn save_harness_state(std::fs::create_dir_all(harness_state_dir)?;let state_path = get_harness_state_path(harness_state_dir);let content = format!("{}\n", serde_json::to_string_pretty(state)?);- crate::settings::storage::atomic_write(&state_path, &content)?;+ crate::settings::storage::atomic_write_with(+ &state_path,+ &content,+ crate::settings::storage::AtomicWriteOptions { fsync: true },+ )?;Ok(state_path)}+/// Locked read-modify-write of the harness state file: both writer sides+/// (the kernel and refine) take `{file}.lock` around the reload and the save.+///+/// # Errors+///+/// Error when the directory cannot be created, the state lock cannot be acquired, or the atomic write fails.+pub fn update_harness_state<R>(+ harness_state_dir: &Path,+ scope: HarnessScope,+ update: impl FnOnce(&mut HarnessState) -> R,+) -> anyhow::Result<(R, PathBuf)> {+ std::fs::create_dir_all(harness_state_dir)?;+ let state_path = get_harness_state_path(harness_state_dir);+ let lock = crate::platform::lock_dir::LockDir::acquire_owned_retrying(+ &state_path,+ std::time::Duration::from_secs(10),+ 50,+ std::time::Duration::from_millis(20),+ )?;+ let mut state = load_harness_state(harness_state_dir, scope);+ let result = update(&mut state);+ lock.ensure_owned()?;+ let written = save_harness_state(harness_state_dir, &state)?;+ Ok((result, written))+}+#[must_use]pub fn get_refinement_history_path(harness_state_dir: &Path) -> PathBuf {harness_state_dir.join(REFINEMENT_HISTORY_FILE_NAME)@@ -636,11 +666,8 @@ mod tests {);}- /// Served-path oracle (refinement.ts:404 passes only `{ mode }` — the- /// measured signal of record 20260928-172400): the save takes NO fsync- /// branch, landing exactly `to_string_pretty(state) + "\n"` bytes.#[test]- fn harness_save_takes_the_ts_default_no_sync() {+ fn harness_save_takes_exactly_one_fsync() {let tmp = tempfile::tempdir().unwrap();let dir = get_global_harness_state_dir(tmp.path());let mut state = empty_harness_state();@@ -657,8 +684,8 @@ mod tests {let written = save_harness_state(&dir, &state).unwrap();assert_eq!(crate::settings::storage::opt_in_fsync_calls(),- before,- "the TS-default harness save must not sync"+ before + 1,+ "the harness save must flush the temp file before the rename");assert_eq!(written, get_harness_state_path(&dir));assert_eq!(@@ -667,6 +694,33 @@ mod tests {);}+ #[test]+ fn concurrent_update_harness_state_writes_all_land() {+ let tmp = tempfile::tempdir().unwrap();+ let dir = get_global_harness_state_dir(tmp.path());+ std::thread::scope(|scope| {+ for worker in 0..8u32 {+ let dir = &dir;+ scope.spawn(move || {+ for i in 0..10u32 {+ let id = format!("m-{worker}-{i}");+ update_harness_state(dir, HarnessScope::Global, |state| {+ let memories = state.entries.get_mut(&RefinementKind::Memory).unwrap();+ memories.insert(+ id.clone(),+ entry(&id, RefinementKind::Memory, HarnessScope::Global, "a fact"),+ );+ memories.len()+ })+ .unwrap();+ }+ });+ }+ });+ let loaded = load_harness_state(&dir, HarnessScope::Global);+ assert_eq!(loaded.entries[&RefinementKind::Memory].len(), 80);+ }+#[test]fn merge_prefixed_local_conflicts() {let mut global = empty_harness_state();diff --git a/crates/pa-core/src/session_engine/host_requests.rs b/crates/pa-core/src/session_engine/host_requests.rsindex 68981f02bc..8c1ec24418 100644--- a/crates/pa-core/src/session_engine/host_requests.rs+++ b/crates/pa-core/src/session_engine/host_requests.rs@@ -306,7 +306,7 @@ pub fn handle_rlm_heartbeat_host_request(let Some(id) = record.get("id").and_then(Value::as_str) else {anyhow::bail!("rlm_heartbeat.delete id must be a string");};- let heartbeat = store.delete_rlm_heartbeat(active_session_id, id, now);+ let heartbeat = store.delete_rlm_heartbeat(active_session_id, id, now)?;Ok(RlmHeartbeatHostOutcome {response: json!({"heartbeat": heartbeatdiff --git a/crates/pa-core/src/session_engine/refine.rs b/crates/pa-core/src/session_engine/refine.rsindex 2c8a6e245e..90b8747908 100644--- a/crates/pa-core/src/session_engine/refine.rs+++ b/crates/pa-core/src/session_engine/refine.rs@@ -14,7 +14,7 @@ use crate::refinement::executor::{};use crate::refinement::{append_global_refinement, format_refinement_notice_body, load_global_refinement_history,- load_harness_state, merge_harness_states, save_harness_state, HarnessScope, RefinementResult,+ load_harness_state, merge_harness_states, update_harness_state, HarnessScope, RefinementResult,};use crate::session::manager::SessionManager;@@ -374,7 +374,6 @@ pub async fn execute_refinement_with_rows(HarnessScope::Global => global_harness_dir.to_path_buf(),HarnessScope::Local => local_harness_dir.clone(),};- let mut state = load_harness_state(&target_dir, target_scope);// The factory opt-in resolves HERE — immediately before the apply,// after the planning request — so a setting that changed during the// request (`/factory off` mid-plan) decides, not a snapshot captured@@ -393,16 +392,20 @@ pub async fn execute_refinement_with_rows(}None => false,};- let mut result = apply_refinement_plan(- &mut state,- plan,- &core_options,- Some(baseline_state),- factory_enabled,- );- result.harness_state_path = save_harness_state(&target_dir, &state)?- .to_string_lossy()- .to_string();+ let (mut result, state_path) = tokio::task::spawn_blocking(move || {+ update_harness_state(&target_dir, target_scope, |state| {+ apply_refinement_plan(+ state,+ plan,+ &core_options,+ Some(baseline_state),+ factory_enabled,+ )+ })+ })+ .await+ .map_err(|error| anyhow::anyhow!("refinement harness update task failed: {error}"))??;+ result.harness_state_path = state_path.to_string_lossy().to_string();if target_scope == HarnessScope::Global {append_global_refinement(global_harness_dir, &result)?;}diff --git a/crates/pa-core/src/settings/storage.rs b/crates/pa-core/src/settings/storage.rsindex 2270a8dac7..ebb251705f 100644--- a/crates/pa-core/src/settings/storage.rs+++ b/crates/pa-core/src/settings/storage.rs@@ -249,7 +249,7 @@ impl SettingsStorage for FileSettingsStorage {}}if let Some(content) = next {- atomic_write(path, &content)?;+ atomic_write_with(path, &content, AtomicWriteOptions { fsync: true })?;}}drop(held);@@ -302,6 +302,9 @@ pub fn atomic_write_with(path: &Path, content: &str, options: AtomicWriteOptions}}crate::platform::rename_onto(&temp, path)?;+ if options.fsync {+ crate::platform::fs::sync_directory(path.parent().unwrap_or_else(|| Path::new(".")))?;+ }Ok(())}@@ -399,7 +402,7 @@ mod tests {}#[test]- fn settings_write_takes_the_ts_default_no_sync() {+ fn settings_write_takes_exactly_one_fsync() {let dir = tempfile::tempdir().unwrap();let storage = FileSettingsStorage::new(dir.path().join("cwd"), dir.path().join("agent"));let document = "{ \"defaultProvider\": \"prime-inference\" }";@@ -407,7 +410,11 @@ mod tests {storage.with_lock(SettingsScope::Global, &mut |_| Some(document.to_string())).unwrap();- assert_eq!(opt_in_fsync_calls(), before);+ assert_eq!(+ opt_in_fsync_calls(),+ before + 1,+ "the settings write must flush the temp file before the rename"+ );let path = dir.path().join("agent").join("settings.json");assert_eq!(fs::read_to_string(&path).unwrap(), document);}diff --git a/crates/pa-core/src/update/download.rs b/crates/pa-core/src/update/download.rsindex 72110126fe..6d63033bfb 100644--- a/crates/pa-core/src/update/download.rs+++ b/crates/pa-core/src/update/download.rs@@ -306,7 +306,7 @@ fn update_install_source(release_dir: &Path, install_source: &str) -> Result<()>}std::fs::write(&path, install_source).with_context(|| format!("write {}", path.display()))?;std::fs::File::open(&path)?.sync_all()?;- super::install::sync_directory(release_dir)+ Ok(super::install::sync_directory(release_dir)?)}/// Copy or extract the payload into staging scratch, check the release@@ -395,7 +395,7 @@ fn sync_release_tree(root: &Path) -> Result<()> {for directory in directories.iter().rev() {super::install::sync_directory(directory)?;}- super::install::sync_directory(root)+ Ok(super::install::sync_directory(root)?)}/// Validate a staged payload, write the installer metadata, and rename it intodiff --git a/crates/pa-core/src/update/install.rs b/crates/pa-core/src/update/install.rsindex 477b00dc20..d33abd5e82 100644--- a/crates/pa-core/src/update/install.rs+++ b/crates/pa-core/src/update/install.rs@@ -204,18 +204,7 @@ pub fn install_source_is_valid(source: &str) -> bool {protocol == "https" || protocol == "http"}-/// fsync a directory so its entries survive a crash (staged releases and-/// the launcher repoint must never reference an unwritten inode; a-/// directory fd opened read-only accepts `sync_all` on POSIX platforms).-///-/// # Errors-///-/// Returns an error when the directory cannot be opened or its `sync_all` fails.-pub fn sync_directory(path: &Path) -> Result<()> {- let dir = std::fs::File::open(path)?;- dir.sync_all()- .with_context(|| format!("fsync {}", path.display()))-}+pub use crate::platform::fs::sync_directory;/// The parsed shape of a launcher link target.struct ReleaseLink {diff --git a/crates/pa-daemon/src/agent_engine/goalcore.rs b/crates/pa-daemon/src/agent_engine/goalcore.rsindex 660bb9b855..31c2300e18 100644--- a/crates/pa-daemon/src/agent_engine/goalcore.rs+++ b/crates/pa-daemon/src/agent_engine/goalcore.rs@@ -137,7 +137,9 @@ impl AgentSessionEngine {{continue;}- let _ = wiring.store.cancel(&job.id, crate::util::now_ms());+ if let Err(error) = wiring.store.cancel(&job.id, crate::util::now_ms()) {+ eprintln!("failed to cancel goal wake job: {error}");+ }}}diff --git a/crates/pa-daemon/src/agent_engine/turn/quota.rs b/crates/pa-daemon/src/agent_engine/turn/quota.rsindex 2a9acbb0c5..83ae59f861 100644--- a/crates/pa-daemon/src/agent_engine/turn/quota.rs+++ b/crates/pa-daemon/src/agent_engine/turn/quota.rs@@ -75,7 +75,9 @@ impl AgentSessionEngine {.iter().any(|job| job.id == job_id && job.status == pa_core::cron::JobStatus::Active);if matches_job {- let _ = wiring.store.cancel(job_id, crate::util::now_ms());+ if let Err(error) = wiring.store.cancel(job_id, crate::util::now_ms()) {+ eprintln!("failed to cancel quota resume job: {error}");+ }}}diff --git a/crates/pa-daemon/src/branch_navigation.rs b/crates/pa-daemon/src/branch_navigation.rsindex a90c62e80e..a873a5c8a3 100644--- a/crates/pa-daemon/src/branch_navigation.rs+++ b/crates/pa-daemon/src/branch_navigation.rs@@ -566,7 +566,9 @@ impl Worker {// source branch.self.refresh_replaced_session_state().await;self.reseed_service_tier_for_replacement();- self.bind_scheduled_jobs().await;+ if let Err(error) = self.bind_scheduled_jobs().await {+ return response_failure(None, "fork", &error.to_string(), None);+ }self.prewarm_replacement_session();// The fork swap is a whole-session replacement too: the fresh summary ships// immediately.diff --git a/crates/pa-daemon/src/scheduled_jobs.rs b/crates/pa-daemon/src/scheduled_jobs.rsindex c280f652b2..4e009717d6 100644--- a/crates/pa-daemon/src/scheduled_jobs.rs+++ b/crates/pa-daemon/src/scheduled_jobs.rs@@ -80,7 +80,7 @@ impl QueueHooks {/// The failed-runnable cancel: the store cancels the dead session's whole/// job set by file, so the artifact never re-fires.- fn cancel_jobs_for_dead_target(&self, job: &AgentCronJob) {+ fn cancel_jobs_for_dead_target(&self, job: &AgentCronJob) -> anyhow::Result<()> {self.store.cancel_jobs_for_session(&CancelJobsFilter {active_session_id: None,@@ -88,7 +88,8 @@ impl QueueHooks {session_file: Some(job.session_file.clone()),},crate::util::now_ms(),- );+ )?;+ Ok(())}}@@ -97,7 +98,7 @@ impl AgentCronSchedulerHooks for QueueHooks {// A persisted job whose target is no longer live (killed — `archived` — or deleted)// cancels the session's jobs and skips: a fire can never revive a stopped session.if Self::persisted_target_gone(job) {- self.cancel_jobs_for_dead_target(job);+ self.cancel_jobs_for_dead_target(job)?;return Ok(Some("skipped"));}let activity = self.activity();@@ -260,19 +261,19 @@ impl ScheduledJobs {&self,binding: SessionBinding,artifact_dir: Option<PathBuf>,- ) {+ ) -> anyhow::Result<()> {if let Some(dir) = artifact_dir {- let _ = std::fs::create_dir_all(&dir);+ std::fs::create_dir_all(&dir)?;self.store.register_session_artifact(&binding.session_id, &dir);}if !binding.session_file.is_empty() {- self.store.rebind_session_jobs(&binding);+ self.store.rebind_session_jobs(&binding)?;}let mut guard = self.scheduler.lock().await;if let Some(scheduler) = guard.as_ref() {scheduler.wake().await;- return;+ return Ok(());}let scheduler = Arc::new(AgentCronScheduler::new(Arc::clone(&self.store),@@ -280,6 +281,7 @@ impl ScheduledJobs {));scheduler.start().await;*guard = Some(scheduler);+ Ok(())}/// Re-arm the timer after a catalog mutation (TS `cronScheduler.wake`).@@ -374,7 +376,7 @@ impl Worker {/// The killed close's cancel: the session's whole job set cancels by any of/// its three identities — durably, so the session's own heartbeats can/// never revive it.- pub(crate) async fn cancel_session_scheduled_jobs(&self) {+ pub(crate) async fn cancel_session_scheduled_jobs(&self) -> anyhow::Result<()> {let (active_session_id, session_id, session_file) = {let core = self.core@@ -382,7 +384,7 @@ impl Worker {.unwrap_or_else(std::sync::PoisonError::into_inner);self.bind_store_artifact(&core);let Some(store) = core.store.as_ref() else {- return;+ return Ok(());};(core.active_session_id.clone(),@@ -397,18 +399,19 @@ impl Worker {session_file: Some(session_file),},crate::util::now_ms(),- );+ )?;for job in &cancelled {self.scheduled.remove_queued_heartbeat_follow_up(job);}if !cancelled.is_empty() {self.scheduled.wake().await;}+ Ok(())}/// Only a subagent's RLM heartbeat jobs cancel here (the plain cron jobs/// survive the replacement); a top-level session cancels nothing.- pub(crate) async fn cancel_session_rlm_heartbeats(&self) {+ pub(crate) async fn cancel_session_rlm_heartbeats(&self) -> anyhow::Result<()> {let (is_subagent, active_session_id) = {let core = self.core@@ -421,18 +424,19 @@ impl Worker {)};if !is_subagent {- return;+ return Ok(());}let cancelled = self.scheduled.store()- .cancel_rlm_heartbeats_for_session(&active_session_id, crate::util::now_ms());+ .cancel_rlm_heartbeats_for_session(&active_session_id, crate::util::now_ms())?;for job in &cancelled {self.scheduled.remove_queued_heartbeat_follow_up(job);}if !cancelled.is_empty() {self.scheduled.wake().await;}+ Ok(())}/// The saved-session delete's cancel: cancel the deleted file's whole job@@ -458,14 +462,16 @@ impl Worker {self.scheduled.store().register_session_artifact(session_id, &dir);- self.scheduled.store().cancel_jobs_for_session(+ if let Err(error) = self.scheduled.store().cancel_jobs_for_session(&pa_core::cron::store::CancelJobsFilter {active_session_id: None,session_id: None,session_file: Some(session_file.to_string_lossy().to_string()),},crate::util::now_ms(),- );+ ) {+ eprintln!("failed to cancel deleted session jobs: {error}");+ }}}diff --git a/crates/pa-daemon/src/scheduled_jobs/arms.rs b/crates/pa-daemon/src/scheduled_jobs/arms.rsindex 381385a0cc..82382a0e5e 100644--- a/crates/pa-daemon/src/scheduled_jobs/arms.rs+++ b/crates/pa-daemon/src/scheduled_jobs/arms.rs@@ -218,7 +218,7 @@ impl Worker {.store().cancel(&job_id, crate::util::now_ms()){- Some(job) => {+ Ok(Some(job)) => {self.scheduled.remove_queued_heartbeat_follow_up(&job);self.scheduled.wake().await;response_success(@@ -227,12 +227,13 @@ impl Worker {Some(json!({ "job": serde_json::to_value(&job).unwrap_or(Value::Null) })),)}- None => response_failure(+ Ok(None) => response_failure(None,"cron_cancel",&format!("No cron job found: {job_id}"),None,),+ Err(error) => response_failure(None, "cron_cancel", &error.to_string(), None),}}@@ -364,20 +365,20 @@ impl Worker {self.bind_store_artifact(&core);}let outcome = match payload.get("action").and_then(Value::as_str) {- Some("pause") => Ok(self+ Some("pause") => self.scheduled.store()- .pause_heartbeat(&active_session_id, now)),+ .pause_heartbeat(&active_session_id, now),Some("resume") => self.scheduled.store().resume_heartbeat(&active_session_id, now),// TS `updateHeartbeatForState`: anything but pause/resume clears the// heartbeat.- _ => Ok(self+ _ => self.scheduled.store()- .clear_heartbeat(&active_session_id, now)),+ .clear_heartbeat(&active_session_id, now),};let outcome = match outcome {Ok(job) => job,diff --git a/crates/pa-daemon/src/scheduling_catalog.rs b/crates/pa-daemon/src/scheduling_catalog.rsindex 7d38f50d21..dbd0479dc9 100644--- a/crates/pa-daemon/src/scheduling_catalog.rs+++ b/crates/pa-daemon/src/scheduling_catalog.rs@@ -777,16 +777,32 @@ impl Supervisor {.find(|passive| passive.job.id == *job_id);if let Some(passive) = passive {let store = Self::passive_job_store(&passive.info);- if let Some(job) = store.cancel(job_id, crate::util::now_ms()) {- self.broadcast_heartbeats_changed();- return (- vec![response_line(&response_success(- Some(command_id),- type_name,- Some(json!({ "job": serde_json::to_value(&job).unwrap_or(Value::Null) })),- ))],- false,- );+ match store.cancel(job_id, crate::util::now_ms()) {+ Ok(Some(job)) => {+ self.broadcast_heartbeats_changed();+ return (+ vec![response_line(&response_success(+ Some(command_id),+ type_name,+ Some(+ json!({ "job": serde_json::to_value(&job).unwrap_or(Value::Null) }),+ ),+ ))],+ false,+ );+ }+ Ok(None) => {}+ Err(error) => {+ return (+ vec![response_line(&response_failure(+ Some(command_id),+ type_name,+ &error.to_string(),+ None,+ ))],+ false,+ );+ }}}(diff --git a/crates/pa-daemon/src/session_navigation.rs b/crates/pa-daemon/src/session_navigation.rsindex 60359d3ed5..c9bc3675a5 100644--- a/crates/pa-daemon/src/session_navigation.rs+++ b/crates/pa-daemon/src/session_navigation.rs@@ -316,7 +316,9 @@ impl Worker {Ok(()) => {self.refresh_replaced_session_state().await;self.reseed_service_tier_for_replacement();- self.bind_scheduled_jobs().await;+ if let Err(error) = self.bind_scheduled_jobs().await {+ return response_failure(None, command, &error.to_string(), None);+ }self.prewarm_replacement_session();// The replacement never pushed a roster delta, so the subscribed// surfaces kept the PREVIOUS session's numbers.diff --git a/crates/pa-daemon/src/stop_cleanup.rs b/crates/pa-daemon/src/stop_cleanup.rsindex ac731bf54d..bca4225cda 100644--- a/crates/pa-daemon/src/stop_cleanup.rs+++ b/crates/pa-daemon/src/stop_cleanup.rs@@ -100,7 +100,7 @@ pub(crate) fn cancel_scheduled_jobs_for_tree(sessions_dir: &Path,root_session_file: &Path,live_files: &HashSet<String>,-) -> usize {+) -> Result<usize> {let members = session_tree(agent_dir, sessions_dir, root_session_file);if members.iter().any(|member| {live_files.contains(@@ -109,7 +109,7 @@ pub(crate) fn cancel_scheduled_jobs_for_tree(.to_string(),)}) {- return 0;+ return Ok(0);}let store = AgentCronJobStore::for_session_artifacts();let mut registered = false;@@ -127,7 +127,7 @@ pub(crate) fn cancel_scheduled_jobs_for_tree(registered = true;}if !registered {- return 0;+ return Ok(0);}let now = crate::util::now_ms();let mut cancelled = 0;@@ -140,10 +140,10 @@ pub(crate) fn cancel_scheduled_jobs_for_tree(session_file: Some(member.session_file.to_string_lossy().to_string()),},now,- )+ )?.len();}- cancelled+ Ok(cancelled)}/// The killed session's `archived` state belt: when the worker died before its close@@ -176,6 +176,10 @@ pub(crate) fn finalize_archived_stop() -> (usize, Option<anyhow::Error>) {let cancelled =cancel_scheduled_jobs_for_tree(agent_dir, sessions_dir, root_session_file, live_files);+ let (cancelled, cancel_error) = match cancelled {+ Ok(count) => (count, None),+ Err(error) => (0, Some(error)),+ };// A live worker covering the tree owns its stores again — including the session// file's state — so the belt skips covered trees like the cancel does.let covered = session_tree(agent_dir, sessions_dir, root_session_file)@@ -188,10 +192,10 @@ pub(crate) fn finalize_archived_stop()});if covered {- return (cancelled, None);+ return (cancelled, cancel_error);}let archive_error = ensure_archived_state(root_session_file).err();- (cancelled, archive_error)+ (cancelled, cancel_error.or(archive_error))}/// The kill route's deleted-child finalize (the `rlmLedgerDelete` marker):@@ -474,6 +478,13 @@ impl Supervisor {Path::new(&root_session_file),&live,);+ let cancelled = match cancelled {+ Ok(count) => count,+ Err(error) => {+ self.log_line(&format!("owned stop: cron cancel failed: {error}"));+ return;+ }+ };if cancelled > 0 {self.log_line(&format!("owned stop: cancelled {cancelled} scheduled job(s) of {root_session_file}"diff --git a/crates/pa-daemon/src/worker/commands.rs b/crates/pa-daemon/src/worker/commands.rsindex ac794c0193..0bc684187b 100644--- a/crates/pa-daemon/src/worker/commands.rs+++ b/crates/pa-daemon/src/worker/commands.rs@@ -567,10 +567,13 @@ impl Worker {// cron jobs but cancels the subagent's RLM heartbeats; `shutdown` keeps them// all. The store cancel is durable, so the stopped session's own heartbeats// can never revive it.- match reason {+ let cancel_result = match reason {KillCloseReason::Killed => self.cancel_session_scheduled_jobs().await,KillCloseReason::Replaced => self.cancel_session_rlm_heartbeats().await,- KillCloseReason::Shutdown => {}+ KillCloseReason::Shutdown => Ok(()),+ };+ if let Err(error) = cancel_result {+ return response_failure(None, "kill", &error.to_string(), None);}// The close cascades to the resident children with the SAME reason before// the session's own archive and dispose; a close failure is swallowed.diff --git a/crates/pa-daemon/src/worker/create.rs b/crates/pa-daemon/src/worker/create.rsindex 53ed743edd..493a9b5fd1 100644--- a/crates/pa-daemon/src/worker/create.rs+++ b/crates/pa-daemon/src/worker/create.rs@@ -586,7 +586,9 @@ impl Worker {}// Bind the schedule catalog onto the session (artifact partition,// job rebind, scheduler start) — TS `rebindCronJobsToState`.- self.bind_scheduled_jobs().await;+ if let Err(error) = self.bind_scheduled_jobs().await {+ return response_failure(None, "create", &error.to_string(), None);+ }// Recovery journal writes must not happen while holding the core// lock: record_recovery locks the core to read the store.let _ = self.record_recovery(true, "create");diff --git a/crates/pa-daemon/src/worker/lifecycle.rs b/crates/pa-daemon/src/worker/lifecycle.rsindex b88f4eb868..8fd260f184 100644--- a/crates/pa-daemon/src/worker/lifecycle.rs+++ b/crates/pa-daemon/src/worker/lifecycle.rs@@ -298,7 +298,7 @@ impl Worker {/// Bind the live session's schedule catalog: register the artifact/// partition, rebind the stored jobs onto the live ids, and start (or/// wake) the scheduler. Runs at create and after every replacement swap.- pub(crate) async fn bind_scheduled_jobs(&self) {+ pub(crate) async fn bind_scheduled_jobs(&self) -> anyhow::Result<()> {let binding = {let core = self.core@@ -307,8 +307,9 @@ impl Worker {crate::scheduled_jobs::live_binding(&core)};if let Some((binding, artifact_dir)) = binding {- self.scheduled.bind_session(binding, artifact_dir).await;+ self.scheduled.bind_session(binding, artifact_dir).await?;}+ Ok(())}/// Clear the queued-input suspension and wake the turn runner so parkeddiff --git a/crates/pa-models/src/cache.rs b/crates/pa-models/src/cache.rsindex dd55378963..b08df0d749 100644--- a/crates/pa-models/src/cache.rs+++ b/crates/pa-models/src/cache.rs@@ -472,7 +472,6 @@ fn write_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> {let mut file = write()?;file.write_all(bytes)?;file.flush()?;- file.sync_all()?;std::fs::rename(&temp, path)}diff --git a/prime-agent-runtime/src/rlm/harness.py b/prime-agent-runtime/src/rlm/harness.pyindex 4d399da3a6..9b43b8dd70 100644--- a/prime-agent-runtime/src/rlm/harness.py+++ b/prime-agent-runtime/src/rlm/harness.py@@ -9,13 +9,17 @@ Execution still belongs to Prime Agent's TypeScript host and the existingfrom __future__ import annotations+import contextlibimport copyimport jsonimport mathimport osimport reimport stat+import sys+import timeimport unicodedata+from collections.abc import Iteratorfrom dataclasses import asdict, dataclass, field, fieldsfrom datetime import datetime, timezonefrom pathlib import Path@@ -24,6 +28,9 @@ from typing import Any, Literalfrom .factory import require_factory_enabled, validate_factory_spec+if sys.platform == "darwin":+ import fcntl+HarnessKind = Literal["prompt", "memory", "skill", "subagent", "factory"]HarnessScope = Literal["local", "global"]@@ -31,6 +38,10 @@ _DEFAULT_FILE_NAME = "harness_state.json"_DEFAULT_HARNESS_DIR_NAME = "harness"_KINDS: tuple[HarnessKind, ...] = ("prompt", "memory", "skill", "subagent", "factory")_state_cache: dict[tuple[Path, HarnessScope], "HarnessState"] = {}+_LOCK_STALE_AFTER = 10.0+_LOCK_ATTEMPTS = 50+_LOCK_RETRY_INTERVAL = 0.02+_PROCESS_TOKEN = uuid4().hexdef _now() -> str:@@ -406,6 +417,7 @@ class HarnessState:# mtime of the file as of the last load/save, used to detect out-of-process# writes (e.g. the host `/refine` command) and avoid clobbering them.self._loaded_mtime: int | None = None+ self._held_lock: tuple[Path, str] | None = Noneself.load()def _ensure_local_writable(self) -> None:@@ -432,6 +444,83 @@ class HarnessState:if self._disk_mtime() != self._loaded_mtime:self.load()+ @staticmethod+ def _owner_dead(recorded: str) -> bool:+ parts = recorded.strip().split(" ")+ if len(parts) != 2 or not parts[1]:+ return True+ try:+ pid = int(parts[0])+ except ValueError:+ return True+ if pid <= 0:+ return True+ try:+ os.kill(pid, 0)+ except ProcessLookupError:+ return True+ except (PermissionError, OSError):+ return False+ return False++ @staticmethod+ def _owner_matches(lock_path: Path, owner: str) -> bool:+ try:+ return (lock_path / "owner").read_text(encoding="utf-8").strip() == owner+ except OSError:+ return False++ def _acquire_lock_dir(self, lock_path: Path) -> str:+ owner = f"{os.getpid()} {_PROCESS_TOKEN}.{uuid4().hex}"+ owner_path = lock_path / "owner"+ for _ in range(_LOCK_ATTEMPTS):+ try:+ os.mkdir(lock_path)+ try:+ os.chmod(lock_path, 0o700)+ owner_path.write_text(f"{owner}\n", encoding="utf-8")+ os.chmod(owner_path, 0o600)+ except OSError:+ lock_path.rmdir()+ raise+ return owner+ except FileExistsError:+ try:+ stale = time.time() - lock_path.stat().st_mtime > _LOCK_STALE_AFTER+ except FileNotFoundError:+ continue+ if stale:+ try:+ recorded = owner_path.read_text(encoding="utf-8")+ except (OSError, UnicodeError):+ recorded = None+ if recorded is None or self._owner_dead(recorded):+ with contextlib.suppress(FileNotFoundError):+ owner_path.unlink()+ with contextlib.suppress(FileNotFoundError):+ lock_path.rmdir()+ continue+ time.sleep(_LOCK_RETRY_INTERVAL)+ raise RuntimeError(f"harness state lock not acquired: {lock_path}")++ @contextlib.contextmanager+ def _locked(self) -> Iterator[None]:+ if self.file_path is not None:+ target = Path(os.path.realpath(self.file_path))+ target.parent.mkdir(parents=True, exist_ok=True)+ lock_path = target.with_name(f"{target.name}.lock")+ self._held_lock = (lock_path, self._acquire_lock_dir(lock_path))+ try:+ self._sync_from_disk()+ yield+ finally:+ if self._held_lock is not None:+ lock_path, owner = self._held_lock+ if self._owner_matches(lock_path, owner):+ (lock_path / "owner").unlink()+ lock_path.rmdir()+ self._held_lock = None+def load(self) -> "HarnessState":if self.file_path is None or not self.file_path.exists():self._loaded_mtime = None@@ -522,6 +611,8 @@ class HarnessState:if self.file_path is None:# in_memory fallback: nothing to persist.return self+ if self._held_lock is None:+ raise RuntimeError("harness state lock not acquired")self.file_path.parent.mkdir(parents=True, exist_ok=True)data = {"schema": 1,@@ -544,9 +635,23 @@ class HarnessState:descriptor = os.open(temp_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)with os.fdopen(descriptor, "w", encoding="utf-8") as f:json.dump(data, f, indent=2, ensure_ascii=False)+ f.flush()+ if sys.platform == "darwin":+ fcntl.fcntl(descriptor, fcntl.F_FULLFSYNC)+ else:+ os.fsync(descriptor)if existing_mode is not None:os.chmod(temp_path, existing_mode)+ lock_path, owner = self._held_lock+ if not self._owner_matches(lock_path, owner):+ raise RuntimeError(f"harness state lock lost: {lock_path}")os.replace(temp_path, target_path)+ if os.name == "posix":+ directory_fd = os.open(target_path.parent, os.O_RDONLY)+ try:+ os.fsync(directory_fd)+ finally:+ os.close(directory_fd)finally:temp_path.unlink(missing_ok=True)self._loaded_mtime = self._disk_mtime()@@ -581,18 +686,18 @@ class HarnessState:source=source,)self._ensure_local_writable()- self._sync_from_disk()- return self._upsert(- kind,- title,- content,- id=id,- path=path,- reference=reference,- arguments=arguments,- metadata=metadata,- source=source,- )+ with self._locked():+ return self._upsert(+ kind,+ title,+ content,+ id=id,+ path=path,+ reference=reference,+ arguments=arguments,+ metadata=metadata,+ source=source,+ )def _upsert(self,@@ -688,14 +793,14 @@ class HarnessState:if target := self._global_target(global_, kwargs):return target.delete(kind, id)self._ensure_local_writable()- self._sync_from_disk()- if kind not in self.entries:- raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")- if id not in self.entries[kind]:- return False- del self.entries[kind][id]- self.save()- return True+ with self._locked():+ if kind not in self.entries:+ raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")+ if id not in self.entries[kind]:+ return False+ del self.entries[kind][id]+ self.save()+ return Truedef list(self, kind: HarnessKind | None = None, *, global_: bool = False, **kwargs: Any) -> list[HarnessEntry]:if target := self._global_target(global_, kwargs):@@ -738,26 +843,26 @@ class HarnessState:source=source,)self._ensure_local_writable()- self._sync_from_disk()- if kind not in self.entries:- raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")- _require_text(kind, _describe_entry(id, title), "title", title)- if id is not None:- _require_text(kind, _describe_entry(id, title), "id", id)- entry_id = id or _slug(title, kind)- if entry_id in self.entries[kind]:- raise ValueError(f"{kind} entry {entry_id!r} already exists")- return self._upsert(- kind,- title,- content,- id=entry_id,- path=path,- reference=reference,- arguments=arguments,- metadata=metadata,- source=source,- )+ with self._locked():+ if kind not in self.entries:+ raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")+ _require_text(kind, _describe_entry(id, title), "title", title)+ if id is not None:+ _require_text(kind, _describe_entry(id, title), "id", id)+ entry_id = id or _slug(title, kind)+ if entry_id in self.entries[kind]:+ raise ValueError(f"{kind} entry {entry_id!r} already exists")+ return self._upsert(+ kind,+ title,+ content,+ id=entry_id,+ path=path,+ reference=reference,+ arguments=arguments,+ metadata=metadata,+ source=source,+ )def update(self,@@ -788,23 +893,23 @@ class HarnessState:source=source,)self._ensure_local_writable()- self._sync_from_disk()- if kind not in self.entries:- raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")- _require_text(kind, _describe_entry(id, title), "id", id)- if id not in self.entries[kind]:- raise ValueError(f"{kind} entry {id!r} does not exist")- return self._upsert(- kind,- title,- content,- id=id,- path=path,- reference=reference,- arguments=arguments,- metadata=metadata,- source=source,- )+ with self._locked():+ if kind not in self.entries:+ raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")+ _require_text(kind, _describe_entry(id, title), "id", id)+ if id not in self.entries[kind]:+ raise ValueError(f"{kind} entry {id!r} does not exist")+ return self._upsert(+ kind,+ title,+ content,+ id=id,+ path=path,+ reference=reference,+ arguments=arguments,+ metadata=metadata,+ source=source,+ )def create_memory(self,@@ -1047,24 +1152,24 @@ class HarnessState:if target := self._global_target(global_, kwargs):return target.record_refinement(trigger, changes, evidence=evidence, outcome=outcome, id=id)self._ensure_local_writable()- self._sync_from_disk()- _validate_refinement_event(trigger, changes, evidence=evidence, outcome=outcome)- if id is not None and (not isinstance(id, str) or not id):- raise ValueError(- f"refinement event rejected: id must be a non-empty string when provided, got {_type_name(id)}"+ with self._locked():+ _validate_refinement_event(trigger, changes, evidence=evidence, outcome=outcome)+ if id is not None and (not isinstance(id, str) or not id):+ raise ValueError(+ f"refinement event rejected: id must be a non-empty string when provided, got {_type_name(id)}"+ )+ event_id = id or f"refine_{len(self.refinements) + 1:04d}"+ normalized_changes = [changes] if isinstance(changes, str) else list(changes)+ event = RefinementEvent(+ id=event_id,+ trigger=trigger,+ changes=normalized_changes,+ evidence=evidence,+ outcome=outcome,)- event_id = id or f"refine_{len(self.refinements) + 1:04d}"- normalized_changes = [changes] if isinstance(changes, str) else list(changes)- event = RefinementEvent(- id=event_id,- trigger=trigger,- changes=normalized_changes,- evidence=evidence,- outcome=outcome,- )- self.refinements.append(event)- self.save()- return event+ self.refinements.append(event)+ self.save()+ return eventdef plan_refinement(self,diff --git a/prime-agent-runtime/test/test_harness.py b/prime-agent-runtime/test/test_harness.pyindex 40b05a9e84..2f010b5f29 100644--- a/prime-agent-runtime/test/test_harness.py+++ b/prime-agent-runtime/test/test_harness.py@@ -2,11 +2,13 @@ from __future__ import annotationsimport jsonimport os+import statimport subprocessimport sysimport tempfileimport unittestfrom pathlib import Path+from unittest.mock import patchfrom rlm import harness as package_harnessfrom rlm import rlm as callable_rlm@@ -523,6 +525,133 @@ class HarnessStateTest(unittest.TestCase):titles = [entry.title for entry in HarnessState(real_path).entries["memory"].values()]self.assertEqual(titles, ["Seed", "Through alias"])+ def test_concurrent_subprocess_creates_all_land(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ state_path = Path(temp_dir) / "harness" / "harness_state.json"+ start = Path(temp_dir) / "start"+ env = dict(os.environ)+ env["PYTHONPATH"] = str(Path(__file__).resolve().parents[1] / "src")+ workers = []+ for worker in range(8):+ code = (+ "import time\n"+ "from pathlib import Path\n"+ "from rlm.harness import HarnessState\n"+ f"while not Path({str(start)!r}).exists():\n"+ " time.sleep(0.005)\n"+ f"state = HarnessState({str(state_path)!r})\n"+ "for i in range(10):\n"+ f" state.create('memory', f'note {worker} {{i}}', f'note {worker} {{i}}', id=f'm-{worker}-{{i}}')\n"+ )+ workers.append(+ subprocess.Popen([sys.executable, "-c", code], env=env)+ )+ start.write_text("", encoding="utf-8")+ for worker in workers:+ self.assertEqual(worker.wait(), 0)+ entries = HarnessState(state_path).list("memory")+ self.assertEqual(len(entries), 80)++ def test_held_lock_dir_defers_writes_and_pins_the_shared_lock_path(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ state_path = Path(temp_dir) / "harness" / "harness_state.json"+ state = HarnessState(state_path)+ state.create("memory", "Seed", "Creates the file.", id="seed")+ lock_dir = Path(f"{os.path.realpath(state_path)}.lock")+ lock_dir.mkdir()+ try:+ with self.assertRaises(RuntimeError) as raised:+ state.create("memory", "Second", "Blocked by the lock.", id="second")+ finally:+ lock_dir.rmdir()+ self.assertIn(str(lock_dir), str(raised.exception))+ entries = HarnessState(state_path).list("memory")+ self.assertEqual([entry.id for entry in entries], ["seed"])++ def test_save_syncs_parent_after_replace(self) -> None:+ if os.name != "posix":+ self.skipTest("directory fsync is POSIX-only")+ with tempfile.TemporaryDirectory() as temp_dir:+ path = Path(temp_dir) / "harness_state.json"+ state = HarnessState(path)+ events: list[str] = []+ replace, fsync = os.replace, os.fsync++ def observed_replace(source: Path, destination: Path) -> None:+ replace(source, destination)+ events.append("replace")++ def observed_fsync(fd: int) -> None:+ fsync(fd)+ events.append("directory sync" if stat.S_ISDIR(os.fstat(fd).st_mode) else "file sync")++ with patch.object(os, "replace", side_effect=observed_replace), patch.object(+ os, "fsync", side_effect=observed_fsync+ ):+ state.create_memory("Durable", "Saved")+ self.assertEqual(events[-2:], ["replace", "directory sync"])++ def test_live_stale_lock_is_not_stolen(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ state_path = Path(temp_dir) / "harness_state.json"+ state = HarnessState(state_path)+ state.create_memory("Seed", "Present")+ lock_dir = Path(f"{os.path.realpath(state_path)}.lock")+ owner = state._acquire_lock_dir(lock_dir)+ os.utime(lock_dir, (1, 1))+ try:+ with self.assertRaises(RuntimeError):+ state.create_memory("Second", "Blocked")+ self.assertTrue(state._owner_matches(lock_dir, owner))+ finally:+ (lock_dir / "owner").unlink()+ lock_dir.rmdir()+ self.assertEqual(len(HarnessState(state_path).list("memory")), 1)++ def test_dead_lock_is_reclaimed_without_old_owner_cleanup(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ state_path = Path(temp_dir) / "harness_state.json"+ state = HarnessState(state_path)+ state.create_memory("Seed", "Present")+ lock_dir = Path(f"{os.path.realpath(state_path)}.lock")+ owner = state._acquire_lock_dir(lock_dir)+ worker = subprocess.Popen([sys.executable, "-c", "pass"])+ worker.wait()+ (lock_dir / "owner").write_text(f"{worker.pid} dead-token\n", encoding="utf-8")+ os.utime(lock_dir, (1, 1))+ state.create_memory("Second", "Saved")+ self.assertFalse(state._owner_matches(lock_dir, owner))+ self.assertFalse(lock_dir.exists())+ self.assertEqual(len(HarnessState(state_path).list("memory")), 2)++ def test_unparseable_stale_owner_is_reclaimed(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ state_path = Path(temp_dir) / "harness_state.json"+ state = HarnessState(state_path)+ state.create_memory("Seed", "Present")+ lock_dir = Path(f"{os.path.realpath(state_path)}.lock")+ lock_dir.mkdir()+ (lock_dir / "owner").write_bytes(b"\xff")+ os.utime(lock_dir, (1, 1))+ state.create_memory("Second", "Saved")+ self.assertFalse(lock_dir.exists())+ self.assertEqual(len(HarnessState(state_path).list("memory")), 2)++ def test_save_refuses_lost_lock(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ state_path = Path(temp_dir) / "harness_state.json"+ state = HarnessState(state_path)+ state.create_memory("Seed", "Present")+ with self.assertRaisesRegex(RuntimeError, "lock lost"):+ with state._locked():+ lock_dir, _ = state._held_lock+ (lock_dir / "owner").write_text("999999 other-token\n", encoding="utf-8")+ state.save()+ self.assertTrue(lock_dir.exists())+ (lock_dir / "owner").unlink()+ lock_dir.rmdir()+ self.assertEqual(len(HarnessState(state_path).list("memory")), 1)+def test_load_ignores_unknown_json_keys(self) -> None:with tempfile.TemporaryDirectory() as temp_dir:state_path = Path(temp_dir) / "harness_state.json"