Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

make on-disk sync and locking match each file's durability class (#3380)

Sebastian Müller97b4a03ffc21
diff --git a/crates/pa-core/src/auth/storage.rs b/crates/pa-core/src/auth/storage.rs
index 4999784931..6db8582eb7 100644
--- a/crates/pa-core/src/auth/storage.rs
+++ b/crates/pa-core/src/auth/storage.rs
@@ -236,7 +236,11 @@ impl AuthStorageBackend for FileAuthStorageBackend {
let current = fs::read_to_string(&self.auth_path).ok();
let ((), next) = update(current)?;
if let Some(next) = next {
- super::super::settings::storage::atomic_write(&self.auth_path, &next)?;
+ super::super::settings::storage::atomic_write_with(
+ &self.auth_path,
+ &next,
+ super::super::settings::storage::AtomicWriteOptions { fsync: true },
+ )?;
}
drop(guard);
Ok(())
@@ -307,10 +311,10 @@ mod tests {
assert!(data.credential("prime-inference").is_some());
}
- /// The auth save goes through the real `with_lock` writer and takes NO fsync branch landing the
- /// exact document bytes.
+ /// The auth save goes through the real `with_lock` writer and takes exactly one fsync branch
+ /// landing the exact document bytes.
#[test]
- fn auth_write_takes_the_ts_default_no_sync() {
+ fn auth_write_takes_exactly_one_fsync() {
let dir = tempfile::tempdir().unwrap();
let backend = FileAuthStorageBackend::new(dir.path().join("auth.json"));
let document = r#"{ "prime-inference": { "type": "api_key", "key": "sk" } }"#;
@@ -320,8 +324,8 @@ mod tests {
.unwrap();
assert_eq!(
crate::settings::storage::opt_in_fsync_calls(),
- before,
- "the TS-default auth write must not sync"
+ before + 1,
+ "the auth write must flush the temp file before the rename"
);
assert_eq!(
fs::read_to_string(dir.path().join("auth.json")).unwrap(),
diff --git a/crates/pa-core/src/cron/scheduler.rs b/crates/pa-core/src/cron/scheduler.rs
index c82a80b933..acd6cb7a53 100644
--- a/crates/pa-core/src/cron/scheduler.rs
+++ b/crates/pa-core/src/cron/scheduler.rs
@@ -89,7 +89,9 @@ impl<H: AgentCronSchedulerHooks + 'static> AgentCronScheduler<H> {
self.core.stopped.store(false, Ordering::SeqCst);
if !self.core.has_started.swap(true, Ordering::SeqCst) {
let now = self.core.hooks.now();
- self.core.store.recover_interrupted_dispatches(now);
+ if let Err(error) = self.core.store.recover_interrupted_dispatches(now) {
+ tracing::warn!(%error, "cron recovery failed at start");
+ }
}
self.schedule_next().await;
}
@@ -113,7 +115,7 @@ impl<H: AgentCronSchedulerHooks + 'static> AgentCronScheduler<H> {
///
/// # Errors
///
- /// The underlying pass never fails in the current implementation.
+ /// Returns an error when a cron state lock or write fails.
pub async fn run_due(&self) -> anyhow::Result<usize> {
self.core.run_due_at(self.core.hooks.now()).await
}
@@ -135,7 +137,7 @@ impl<H: AgentCronSchedulerHooks + 'static> SchedulerCore<H> {
///
/// # Errors
///
- /// The current implementation never returns `Err`.
+ /// Returns an error when a cron state lock or write fails.
pub async fn run_due_at(&self, now: u64) -> anyhow::Result<usize> {
// The pass claim is atomic: exactly one pass runs at a time.
if (self.stopped.load(Ordering::SeqCst) && self.has_started.load(Ordering::SeqCst))
@@ -150,8 +152,8 @@ impl<H: AgentCronSchedulerHooks + 'static> SchedulerCore<H> {
let _running = RunningGuard(&self.running);
// Recover interrupted dispatches before claiming: a claim left
// by an unwound pass is released here.
- self.store.recover_interrupted_dispatches(now);
- let claimed = self.store.claim_due(now, self.hooks.now());
+ self.store.recover_interrupted_dispatches(now)?;
+ let claimed = self.store.claim_due(now, self.hooks.now())?;
let dispatches: Vec<PendingDispatch> = claimed
.into_iter()
.map(|dispatch| {
@@ -203,16 +205,16 @@ impl<H: AgentCronSchedulerHooks + 'static> SchedulerCore<H> {
) -> Option<&'static str> {
let outcome = async {
let Some(job) = self.store.get_claimed_job(&dispatch.job.id) else {
- self.store
- .record_dispatch_result(
- &dispatch.id,
- &DispatchResultOptions {
- now: Some(self.hooks.now()),
- outcome: "skipped",
- error: None,
- },
- )
- .ok();
+ if let Err(error) = self.store.record_dispatch_result(
+ &dispatch.id,
+ &DispatchResultOptions {
+ now: Some(self.hooks.now()),
+ outcome: "skipped",
+ error: None,
+ },
+ ) {
+ tracing::warn!(%error, "cron dispatch result not saved");
+ }
return Some("skipped");
};
let mut run_error: Option<String> = None;
@@ -231,22 +233,25 @@ impl<H: AgentCronSchedulerHooks + 'static> SchedulerCore<H> {
"ran"
};
let failed = run_error.is_some();
- self.store
- .record_dispatch_result(
- &dispatch.id,
- &DispatchResultOptions {
- now: Some(self.hooks.now()),
- outcome,
- error: run_error,
- },
- )
- .ok();
+ if let Err(error) = self.store.record_dispatch_result(
+ &dispatch.id,
+ &DispatchResultOptions {
+ now: Some(self.hooks.now()),
+ outcome,
+ error: run_error,
+ },
+ ) {
+ tracing::warn!(%error, "cron dispatch result not saved");
+ }
if failed {
let now = self.hooks.now();
let streak = self.note_run_failure(&dispatch.job.id);
- let _ = self
+ if let Err(error) = self
.store
- .defer_next_run(&dispatch.job.id, now + failure_backoff_ms(streak));
+ .defer_next_run(&dispatch.job.id, now + failure_backoff_ms(streak))
+ {
+ tracing::warn!(%error, "cron failure backoff not saved");
+ }
} else if outcome == "ran" {
self.clear_run_failure(&dispatch.job.id);
}
@@ -541,7 +546,10 @@ mod tests {
let job = store
.create(&input("tick", "every 10m", now))
.expect("first job");
- store.cancel(&job.id, now).expect("cancel the only job");
+ store
+ .cancel(&job.id, now)
+ .unwrap()
+ .expect("cancel the only job");
let deadline = std::time::Instant::now() + std::time::Duration::from_millis(300);
while std::time::Instant::now() < deadline {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
diff --git a/crates/pa-core/src/cron/store/heartbeat.rs b/crates/pa-core/src/cron/store/heartbeat.rs
index 187d4250f3..7c962cb2f7 100644
--- a/crates/pa-core/src/cron/store/heartbeat.rs
+++ b/crates/pa-core/src/cron/store/heartbeat.rs
@@ -101,11 +101,20 @@ impl AgentCronJobStore {
};
let mut jobs = existing;
jobs.push(job.clone());
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
Ok(job)
}
- pub fn pause_heartbeat(&self, active_session_id: &str, now: u64) -> Option<AgentCronJob> {
- let current = self.get_heartbeat(active_session_id)?;
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
+ pub fn pause_heartbeat(
+ &self,
+ active_session_id: &str,
+ now: u64,
+ ) -> anyhow::Result<Option<AgentCronJob>> {
+ let Some(current) = self.get_heartbeat(active_session_id) else {
+ return Ok(None);
+ };
let now_iso = iso_from_millis(now);
let mut paused = None;
let jobs: Vec<AgentCronJob> = self
@@ -125,8 +134,8 @@ impl AgentCronJobStore {
paused_job
})
.collect();
- self.write_jobs(&jobs);
- paused
+ self.write_jobs(&jobs)?;
+ Ok(paused)
}
/// Resume the heartbeat for a session, recomputing its next run time. `Ok(None)`
@@ -165,12 +174,21 @@ impl AgentCronJobStore {
resumed_job
})
.collect();
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
Ok(resumed)
}
- pub fn clear_heartbeat(&self, active_session_id: &str, now: u64) -> Option<AgentCronJob> {
- let current = self.get_heartbeat(active_session_id)?;
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
+ pub fn clear_heartbeat(
+ &self,
+ active_session_id: &str,
+ now: u64,
+ ) -> anyhow::Result<Option<AgentCronJob>> {
+ let Some(current) = self.get_heartbeat(active_session_id) else {
+ return Ok(None);
+ };
let now_iso = iso_from_millis(now);
let mut cleared = None;
let jobs: Vec<AgentCronJob> = self
@@ -190,8 +208,8 @@ impl AgentCronJobStore {
cleared_job
})
.collect();
- self.write_jobs(&jobs);
- cleared
+ self.write_jobs(&jobs)?;
+ Ok(cleared)
}
/// Apply a pause, stop, or resume management action to a heartbeat job. `Ok(None)`
@@ -263,10 +281,10 @@ impl AgentCronJobStore {
}
})
.collect();
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
return Ok(Some(updated_job));
}
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
Ok(Some(updated_job))
}
}
@@ -276,6 +294,25 @@ mod tests {
use super::*;
use crate::cron::store::input;
+ #[test]
+ fn pause_does_not_ack_when_state_lock_is_held() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("jobs.json");
+ let store = AgentCronJobStore::new(path.clone());
+ let now = 1_700_000_000_000;
+ let heartbeat = store
+ .create_heartbeat(&input("continue", "every 5m", now))
+ .unwrap();
+ let before = std::fs::read(&path).unwrap();
+ let held =
+ crate::platform::lock_dir::LockDir::acquire(&path, std::time::Duration::from_secs(30))
+ .unwrap();
+ assert!(store.pause_heartbeat("live-1", now + 1).is_err());
+ assert_eq!(std::fs::read(&path).unwrap(), before);
+ drop(held);
+ assert_eq!(store.get_heartbeat("live-1"), Some(heartbeat));
+ }
+
#[test]
fn heartbeat_lifecycle() {
let dir = tempfile::TempDir::new().unwrap();
@@ -285,7 +322,7 @@ mod tests {
.create_heartbeat(&input("continue the mission", "every 5m", now))
.unwrap();
assert_eq!(store.get_heartbeat("live-1").unwrap().id, heartbeat.id);
- let paused = store.pause_heartbeat("live-1", now + 1).unwrap();
+ let paused = store.pause_heartbeat("live-1", now + 1).unwrap().unwrap();
assert_eq!(paused.status, JobStatus::Paused);
assert_eq!(paused.next_run_at, None);
let resumed = store.resume_heartbeat("live-1", now + 2).unwrap().unwrap();
@@ -299,7 +336,7 @@ mod tests {
let jobs = store.list();
let cancelled_first = jobs.iter().find(|job| job.id != second.id).unwrap();
assert_eq!(cancelled_first.status, JobStatus::Cancelled);
- let cleared = store.clear_heartbeat("live-1", now + 4).unwrap();
+ let cleared = store.clear_heartbeat("live-1", now + 4).unwrap().unwrap();
assert_eq!(cleared.status, JobStatus::Cancelled);
assert!(store
.create_heartbeat(&input("nope", "in 10m", now))
diff --git a/crates/pa-core/src/cron/store/jobs.rs b/crates/pa-core/src/cron/store/jobs.rs
index fce3c0b553..829d04a786 100644
--- a/crates/pa-core/src/cron/store/jobs.rs
+++ b/crates/pa-core/src/cron/store/jobs.rs
@@ -62,13 +62,17 @@ impl AgentCronJobStore {
};
let mut jobs = self.read_jobs();
jobs.push(job.clone());
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
Ok(job)
}
/// Bind jobs stored for a session file to a live session id on restore, or
/// move a live session's jobs to a new file when it switches.
- pub fn rebind_session_jobs(&self, input: &SessionBinding) -> Vec<AgentCronJob> {
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
+ pub fn rebind_session_jobs(&self, input: &SessionBinding) -> anyhow::Result<Vec<AgentCronJob>> {
let target_session_file = resolve_path(&input.session_file);
let mut rebound_jobs = Vec::new();
let jobs: Vec<AgentCronJob> = self
@@ -99,11 +103,18 @@ impl AgentCronJobStore {
})
.collect();
if !rebound_jobs.is_empty() {
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
}
- rebound_jobs
+ Ok(rebound_jobs)
}
- pub fn cancel_jobs_for_session(&self, input: &CancelJobsFilter, now: u64) -> Vec<AgentCronJob> {
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
+ pub fn cancel_jobs_for_session(
+ &self,
+ input: &CancelJobsFilter,
+ now: u64,
+ ) -> anyhow::Result<Vec<AgentCronJob>> {
let now_iso = iso_from_millis(now);
let target_session_file = input.session_file.as_ref().map(|file| resolve_path(file));
let mut cancelled = Vec::new();
@@ -136,11 +147,14 @@ impl AgentCronJobStore {
})
.collect();
if !cancelled.is_empty() {
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
}
- cancelled
+ Ok(cancelled)
}
- pub fn cancel(&self, id: &str, now: u64) -> Option<AgentCronJob> {
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
+ pub fn cancel(&self, id: &str, now: u64) -> anyhow::Result<Option<AgentCronJob>> {
let now_iso = iso_from_millis(now);
let mut cancelled = None;
let jobs: Vec<AgentCronJob> = self
@@ -161,9 +175,9 @@ impl AgentCronJobStore {
})
.collect();
if cancelled.is_some() {
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
}
- cancelled
+ Ok(cancelled)
}
/// Record one run: bump counters, roll `nextRunAt`, complete one-shots.
@@ -216,13 +230,17 @@ impl AgentCronJobStore {
})
.collect();
if updated.is_some() {
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
}
Ok(updated)
}
/// Record a skipped run: roll `nextRunAt`, stamp `lastSkippedAt`.
- pub fn record_skip_result(&self, id: &str, now: u64) -> Option<AgentCronJob> {
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
+ pub fn record_skip_result(&self, id: &str, now: u64) -> anyhow::Result<Option<AgentCronJob>> {
let now_iso = iso_from_millis(now);
let mut updated = None;
let jobs: Vec<AgentCronJob> = self
@@ -247,9 +265,9 @@ impl AgentCronJobStore {
})
.collect();
if updated.is_some() {
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
}
- updated
+ Ok(updated)
}
/// Push a job's next run later, never earlier (the scheduler's consecutive-failure backoff); an
@@ -258,7 +276,11 @@ impl AgentCronJobStore {
/// Runs under the store's state locks: a cancel landing between the failure and this
/// defer must stay cancelled, so the status check and the write are one atomic
/// state mutation.
- pub fn defer_next_run(&self, id: &str, until_ms: u64) -> Option<AgentCronJob> {
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
+ pub fn defer_next_run(&self, id: &str, until_ms: u64) -> anyhow::Result<Option<AgentCronJob>> {
let until_iso = iso_from_millis(until_ms);
let mut updated = None;
self.mutate_states(|state| {
@@ -282,8 +304,8 @@ impl AgentCronJobStore {
})
.collect();
Vec::new()
- });
- updated
+ })?;
+ Ok(updated)
}
pub fn due(&self, now: u64) -> Vec<AgentCronJob> {
@@ -294,7 +316,15 @@ impl AgentCronJobStore {
}
/// Atomically claim due jobs: advance their schedule and record dispatches.
- pub fn claim_due(&self, due_at: u64, claimed_at: u64) -> Vec<AgentCronDispatch> {
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
+ pub fn claim_due(
+ &self,
+ due_at: u64,
+ claimed_at: u64,
+ ) -> anyhow::Result<Vec<AgentCronDispatch>> {
self.mutate_states(|state| claim_due_in_state(state, due_at, claimed_at))
}
@@ -320,7 +350,7 @@ impl AgentCronJobStore {
///
/// # Errors
///
- /// The current implementation never returns `Err`.
+ /// Returns an error when the state lock or write fails.
pub fn record_dispatch_result(
&self,
dispatch_id: &str,
@@ -377,32 +407,38 @@ impl AgentCronJobStore {
})
.collect();
Vec::new()
- });
+ })?;
Ok(updated)
}
- pub fn recover_interrupted_dispatches(&self, now: u64) -> Vec<AgentCronJob> {
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
+ pub fn recover_interrupted_dispatches(&self, now: u64) -> anyhow::Result<Vec<AgentCronJob>> {
let mut recovered = Vec::new();
self.mutate_states(|state| {
recover_interrupted_in_state(state, now, &mut recovered, None);
Vec::new()
- });
- recovered
+ })?;
+ Ok(recovered)
}
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
pub fn recover_interrupted_dispatches_by_id(
&self,
dispatch_ids: &[String],
now: u64,
- ) -> Vec<AgentCronJob> {
+ ) -> anyhow::Result<Vec<AgentCronJob>> {
let mut recovered = Vec::new();
let dispatch_ids: std::collections::HashSet<String> =
dispatch_ids.iter().cloned().collect();
self.mutate_states(|state| {
recover_interrupted_in_state(state, now, &mut recovered, Some(&dispatch_ids));
Vec::new()
- });
- recovered
+ })?;
+ Ok(recovered)
}
pub fn get_due_job(&self, id: &str, now: u64) -> Option<AgentCronJob> {
@@ -439,20 +475,37 @@ mod tests {
assert_eq!(store.list().len(), 1);
let due = store.due(now + 600_000);
assert!(due.iter().any(|found| found.id == job.id));
- let cancelled = store.cancel(&job.id, now + 1).unwrap();
+ let cancelled = store.cancel(&job.id, now + 1).unwrap().unwrap();
assert_eq!(cancelled.status, JobStatus::Cancelled);
assert_eq!(store.list()[0].status, JobStatus::Cancelled);
assert!(store.create(&input(" ", "every 10m", now)).is_err());
}
+ #[test]
+ fn cancel_does_not_ack_when_state_lock_is_held() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("jobs.json");
+ let store = AgentCronJobStore::new(path.clone());
+ let now = 1_700_000_000_000;
+ let job = store.create(&input("tick", "every 10m", now)).unwrap();
+ let before = std::fs::read(&path).unwrap();
+ let held =
+ crate::platform::lock_dir::LockDir::acquire(&path, std::time::Duration::from_secs(30))
+ .unwrap();
+ assert!(store.cancel(&job.id, now + 1).is_err());
+ assert_eq!(std::fs::read(&path).unwrap(), before);
+ drop(held);
+ assert_eq!(store.list(), vec![job]);
+ }
+
#[test]
fn claim_dispatch_and_result_recording() {
let dir = tempfile::TempDir::new().unwrap();
let store = AgentCronJobStore::new(dir.path().join("jobs.json"));
let now = 1_700_000_000_000;
let job = store.create(&input("tick", "every 10m", now)).unwrap();
- assert!(store.claim_due(now, now).is_empty());
- let dispatches = store.claim_due(now + 600_000, now + 600_000);
+ assert!(store.claim_due(now, now).unwrap().is_empty());
+ let dispatches = store.claim_due(now + 600_000, now + 600_000).unwrap();
assert_eq!(dispatches.len(), 1);
assert_eq!(dispatches[0].job.id, job.id);
assert_eq!(
@@ -473,9 +526,11 @@ mod tests {
.unwrap();
assert_eq!(updated.run_count, 1);
assert!(store.get_claimed_job(&job.id).is_none());
- let second = store.claim_due(now + 1_200_000, now + 1_200_000);
+ let second = store.claim_due(now + 1_200_000, now + 1_200_000).unwrap();
assert_eq!(second.len(), 1);
- let recovered = store.recover_interrupted_dispatches(now + 1_300_000);
+ let recovered = store
+ .recover_interrupted_dispatches(now + 1_300_000)
+ .unwrap();
assert_eq!(recovered.len(), 1);
assert_eq!(
recovered[0].last_error.as_deref(),
@@ -522,6 +577,28 @@ mod tests {
assert!(store.list().iter().any(|job| job.last_skipped_at.is_some()));
}
+ #[test]
+ fn claim_due_skips_while_the_state_lock_is_held() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = AgentCronJobStore::new(dir.path().join("jobs.json"));
+ let now = 1_700_000_000_000;
+ let job = store.create(&input("tick", "every 10m", now)).unwrap();
+ let path = store.require_file_path();
+ let before = std::fs::read(&path).unwrap();
+ {
+ let _held = crate::platform::lock_dir::LockDir::acquire(
+ &path,
+ std::time::Duration::from_secs(30),
+ )
+ .unwrap();
+ assert!(store.claim_due(now + 600_000, now + 600_000).is_err());
+ assert_eq!(std::fs::read(&path).unwrap(), before);
+ }
+ let dispatches = store.claim_due(now + 600_000, now + 600_000).unwrap();
+ assert_eq!(dispatches.len(), 1);
+ assert_eq!(dispatches[0].job.id, job.id);
+ }
+
#[test]
fn defer_next_run_pushes_only_later() {
let dir = tempfile::TempDir::new().unwrap();
@@ -529,19 +606,28 @@ mod tests {
let now = 1_700_000_000_000;
let job = store.create(&input("tick", "every 10m", now)).unwrap();
let scheduled_next = now + 600_000;
- assert!(store.defer_next_run(&job.id, now + 60_000).is_none());
+ assert!(store
+ .defer_next_run(&job.id, now + 60_000)
+ .unwrap()
+ .is_none());
let current = store.list().pop().expect("job kept");
assert_eq!(
crate::cron::parse_iso_millis(current.next_run_at.as_deref().unwrap()),
Some(scheduled_next)
);
let deferred = now + 900_000;
- let updated = store.defer_next_run(&job.id, deferred).expect("deferred");
+ let updated = store
+ .defer_next_run(&job.id, deferred)
+ .unwrap()
+ .expect("deferred");
assert_eq!(
crate::cron::parse_iso_millis(updated.next_run_at.as_deref().unwrap()),
Some(deferred)
);
- assert!(store.defer_next_run(&job.id, now + 120_000).is_none());
+ assert!(store
+ .defer_next_run(&job.id, now + 120_000)
+ .unwrap()
+ .is_none());
let current = store.list().pop().expect("job kept");
assert_eq!(
crate::cron::parse_iso_millis(current.next_run_at.as_deref().unwrap()),
@@ -550,6 +636,9 @@ mod tests {
// Cancelled jobs are never deferred. Fresh clock: the jobs file merges on
// `updatedAt` freshness, so a stale stamp would lose the cancel.
store.cancel(&job.id, now_millis()).unwrap();
- assert!(store.defer_next_run(&job.id, now + 2_000_000).is_none());
+ assert!(store
+ .defer_next_run(&job.id, now + 2_000_000)
+ .unwrap()
+ .is_none());
}
}
diff --git a/crates/pa-core/src/cron/store/rlm_heartbeat.rs b/crates/pa-core/src/cron/store/rlm_heartbeat.rs
index 565c9d1be5..912475f123 100644
--- a/crates/pa-core/src/cron/store/rlm_heartbeat.rs
+++ b/crates/pa-core/src/cron/store/rlm_heartbeat.rs
@@ -87,7 +87,7 @@ impl AgentCronJobStore {
};
let mut jobs = self.read_jobs();
jobs.push(job.clone());
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
Ok(job)
}
@@ -169,7 +169,7 @@ impl AgentCronJobStore {
.collect();
if matched {
if let Some(updated_job) = &updated {
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
return Ok(Some(updated_job.clone()));
}
anyhow::bail!(
@@ -179,12 +179,15 @@ impl AgentCronJobStore {
Ok(None)
}
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
pub fn delete_rlm_heartbeat(
&self,
active_session_id: &str,
id: &str,
now: u64,
- ) -> Option<AgentCronJob> {
+ ) -> anyhow::Result<Option<AgentCronJob>> {
let now_iso = iso_from_millis(now);
let mut deleted = None;
let jobs: Vec<AgentCronJob> = self
@@ -208,16 +211,19 @@ impl AgentCronJobStore {
})
.collect();
if deleted.is_some() {
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
}
- deleted
+ Ok(deleted)
}
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
pub fn cancel_rlm_heartbeats_for_session(
&self,
active_session_id: &str,
now: u64,
- ) -> Vec<AgentCronJob> {
+ ) -> anyhow::Result<Vec<AgentCronJob>> {
let now_iso = iso_from_millis(now);
let mut cancelled = Vec::new();
let jobs: Vec<AgentCronJob> = self
@@ -241,9 +247,9 @@ impl AgentCronJobStore {
})
.collect();
if !cancelled.is_empty() {
- self.write_jobs(&jobs);
+ self.write_jobs(&jobs)?;
}
- cancelled
+ Ok(cancelled)
}
}
@@ -289,10 +295,13 @@ mod tests {
assert_eq!(resumed.status, JobStatus::Active);
let deleted = store
.delete_rlm_heartbeat("live-1", &rlm.id, now + 3)
+ .unwrap()
.unwrap();
assert_eq!(deleted.status, JobStatus::Cancelled);
let second = store.create_rlm_heartbeat(&rlm_input).unwrap();
- let cancelled = store.cancel_rlm_heartbeats_for_session("live-1", now + 4);
+ let cancelled = store
+ .cancel_rlm_heartbeats_for_session("live-1", now + 4)
+ .unwrap();
assert_eq!(cancelled.len(), 1);
assert_eq!(second.status, JobStatus::Active);
assert_eq!(cancelled[0].status, JobStatus::Cancelled);
diff --git a/crates/pa-core/src/cron/store/session_artifacts.rs b/crates/pa-core/src/cron/store/session_artifacts.rs
index df6df041d2..8ce5e1273f 100644
--- a/crates/pa-core/src/cron/store/session_artifacts.rs
+++ b/crates/pa-core/src/cron/store/session_artifacts.rs
@@ -42,26 +42,37 @@ impl AgentCronJobStore {
files.clone().into_iter().collect()
}
- pub fn recover_session_artifact(&self, session_id: &str, now: u64) -> Vec<AgentCronJob> {
+ /// # Errors
+ ///
+ /// Returns an error when the state lock or write fails.
+ pub fn recover_session_artifact(
+ &self,
+ session_id: &str,
+ now: u64,
+ ) -> anyhow::Result<Vec<AgentCronJob>> {
let Some(path) = self
.session_artifacts()
.into_iter()
.find(|(id, _)| id == session_id)
.map(|(_, path)| path)
else {
- return Vec::new();
+ return Ok(Vec::new());
};
- with_state_locks(std::slice::from_ref(&path), || {
- let mut state = read_jobs_state(&path);
- let mut recovered = Vec::new();
- if !state.dispatches.is_empty() {
- recover_interrupted_in_state(&mut state, now, &mut recovered, None);
- write_jobs_state(&path, &state);
- }
- recovered
- })
+ with_state_locks(
+ std::slice::from_ref(&path),
+ || -> anyhow::Result<Vec<AgentCronJob>> {
+ let mut state = read_jobs_state(&path);
+ let mut recovered = Vec::new();
+ if !state.dispatches.is_empty() {
+ recover_interrupted_in_state(&mut state, now, &mut recovered, None);
+ write_jobs_state(&path, &state)?;
+ }
+ Ok(recovered)
+ },
+ )
+ .ok_or_else(|| anyhow::anyhow!("cron jobs state lock not acquired; skipped"))?
}
- pub(crate) fn write_jobs_session_artifacts(&self, jobs: &[AgentCronJob]) {
+ pub(crate) fn write_jobs_session_artifacts(&self, jobs: &[AgentCronJob]) -> anyhow::Result<()> {
let artifact_files = self
.session_artifact_files
.lock()
@@ -71,12 +82,14 @@ impl AgentCronJobStore {
artifact_files.keys().cloned().collect();
for job in jobs {
if !registered.contains(&job.session_id) {
- // Mirror the TS error contract.
- return;
+ anyhow::bail!(
+ "cron job session artifact not registered: {}",
+ job.session_id
+ );
}
}
let paths: Vec<PathBuf> = artifact_files.values().cloned().collect();
- with_state_locks(&paths, || {
+ let with_locks = with_state_locks(&paths, || -> anyhow::Result<()> {
let current_by_session_id: HashMap<String, CronJobsState> = artifact_files
.iter()
.map(|(session_id, path)| (session_id.clone(), read_jobs_state(path)))
@@ -136,18 +149,20 @@ impl AgentCronJobStore {
if serde_json::to_string(&current).unwrap_or_default()
!= serde_json::to_string(&next_state).unwrap_or_default()
{
- write_jobs_state(path, &next_state);
+ write_jobs_state(path, &next_state)?;
}
}
+ Ok(())
});
+ with_locks.ok_or_else(|| anyhow::anyhow!("cron jobs state lock not acquired; skipped"))?
}
}
-/// Read one scheduled-jobs artifact file's job rows: a locked, read-only scan A missing or
+/// Read one scheduled-jobs artifact file's job rows. A missing or
/// unparseable file reads as no jobs.
#[must_use]
pub fn read_scheduled_jobs_artifact(path: &Path) -> Vec<AgentCronJob> {
- with_state_locks(&[path.to_path_buf()], || read_jobs_state(path).jobs)
+ read_jobs_state(path).jobs
}
#[cfg(test)]
@@ -178,21 +193,25 @@ mod tests {
assert_eq!(state_a.jobs[0].prompt, "job a");
let state_b = read_jobs_state(&artifacts_b.join(SESSION_SCHEDULED_JOBS_FILENAME));
assert_eq!(state_b.jobs.len(), 1);
- let rebound = store.rebind_session_jobs(&SessionBinding {
- active_session_id: "live-2".to_string(),
- session_id: "session-1".to_string(),
- session_file: "/w/session.jsonl".to_string(),
- cwd: "/w".to_string(),
- });
+ let rebound = store
+ .rebind_session_jobs(&SessionBinding {
+ active_session_id: "live-2".to_string(),
+ session_id: "session-1".to_string(),
+ session_file: "/w/session.jsonl".to_string(),
+ cwd: "/w".to_string(),
+ })
+ .unwrap();
assert_eq!(rebound.len(), 2);
assert!(rebound.iter().all(|job| job.active_session_id == "live-2"));
- let cancelled = store.cancel_jobs_for_session(
- &CancelJobsFilter {
- session_file: Some("/w/session.jsonl".to_string()),
- ..Default::default()
- },
- now + 1,
- );
+ let cancelled = store
+ .cancel_jobs_for_session(
+ &CancelJobsFilter {
+ session_file: Some("/w/session.jsonl".to_string()),
+ ..Default::default()
+ },
+ now + 1,
+ )
+ .unwrap();
// Both jobs share the session file (the rebind moved both), so both cancel.
assert_eq!(cancelled.len(), 2);
}
diff --git a/crates/pa-core/src/cron/store/state.rs b/crates/pa-core/src/cron/store/state.rs
index c0e258fb97..30e85e71bb 100644
--- a/crates/pa-core/src/cron/store/state.rs
+++ b/crates/pa-core/src/cron/store/state.rs
@@ -39,7 +39,7 @@ impl AgentCronJobStore {
pub(crate) fn mutate_states(
&self,
mut mutator: impl FnMut(&mut CronJobsState) -> Vec<AgentCronDispatch>,
- ) -> Vec<AgentCronDispatch> {
+ ) -> anyhow::Result<Vec<AgentCronDispatch>> {
let paths: Vec<PathBuf> = if self.session_artifact_mode {
self.session_artifact_files
.lock()
@@ -59,31 +59,34 @@ impl AgentCronJobStore {
let before = serde_json::to_string(&state).unwrap_or_default();
dispatches.extend(mutator(&mut state));
if serde_json::to_string(&state).unwrap_or_default() != before {
- write_jobs_state(path, &state);
+ write_jobs_state(path, &state)?;
changed = true;
}
}
- dispatches
- });
+ Ok::<_, anyhow::Error>(dispatches)
+ })
+ .ok_or_else(|| anyhow::anyhow!("cron jobs state lock not acquired; skipped"))??;
if changed && Self::heartbeat_catalog_signature(&self.read_jobs()) != previous_heartbeats {
self.notify_heartbeat_change();
}
- dispatches
+ Ok(dispatches)
}
- pub(crate) fn write_jobs(&self, jobs: &[AgentCronJob]) {
+ pub(crate) fn write_jobs(&self, jobs: &[AgentCronJob]) -> anyhow::Result<()> {
let previous_heartbeats = Self::heartbeat_catalog_signature(&self.read_jobs());
if self.session_artifact_mode {
- self.write_jobs_session_artifacts(jobs);
+ self.write_jobs_session_artifacts(jobs)?;
} else {
let path = self.require_file_path();
with_state_locks(std::slice::from_ref(&path), || {
- write_jobs_file(&path, jobs, true);
- });
+ write_jobs_file(&path, jobs, true)
+ })
+ .ok_or_else(|| anyhow::anyhow!("cron jobs state lock not acquired; skipped"))??;
}
if Self::heartbeat_catalog_signature(&self.read_jobs()) != previous_heartbeats {
self.notify_heartbeat_change();
}
+ Ok(())
}
pub(crate) fn require_file_path(&self) -> PathBuf {
@@ -257,7 +260,7 @@ pub(crate) fn recover_interrupted_in_state(
}
/// Cross-process state locks: lockfile with stale takeover, sorted by path.
-pub(crate) fn with_state_locks<T>(paths: &[PathBuf], action: impl FnOnce() -> T) -> T {
+pub(crate) fn with_state_locks<T>(paths: &[PathBuf], action: impl FnOnce() -> T) -> Option<T> {
let mut unique: Vec<PathBuf> = paths.to_vec();
unique.sort();
unique.dedup();
@@ -269,41 +272,26 @@ pub(crate) fn with_state_locks<T>(paths: &[PathBuf], action: impl FnOnce() -> T)
// TS `withCronJobsStateLocks`: proper-lockfile, 100 attempts x
// 10ms, 30s staleness takeover.
let stale = std::time::Duration::from_millis(LOCK_STALE_MS);
- let mut acquired = false;
- let mut failure: Option<std::io::Error> = None;
- for _ in 0..100 {
- match crate::platform::lock_dir::LockDir::acquire(path, stale) {
- Ok(guard) => {
- guards.push(guard);
- acquired = true;
- break;
- }
- Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
- std::thread::sleep(std::time::Duration::from_millis(10));
- }
- Err(error) => {
- failure = Some(error);
- break;
- }
+ match crate::platform::lock_dir::LockDir::acquire_retrying(
+ path,
+ stale,
+ 100,
+ std::time::Duration::from_millis(10),
+ ) {
+ Ok(guard) => guards.push(guard),
+ Err(error) => {
+ tracing::warn!(
+ error = %error,
+ path = %path.display(),
+ "cron jobs state lock not acquired; skipped"
+ );
+ return None;
}
}
- if !acquired {
- // TS throws when the lock is not acquired; the action still runs, but never silently: a
- // concurrent writer may be mutating the file.
- tracing::warn!(
- error = failure.as_ref().map_or_else(
- || "lock still held after retries".to_string(),
- ToString::to_string,
- ),
- path = %path.display(),
- "cron jobs state lock not acquired; running unlocked"
- );
- break;
- }
}
let result = action();
drop(guards);
- result
+ Some(result)
}
pub(crate) fn read_jobs_state(path: &Path) -> CronJobsState {
@@ -338,7 +326,7 @@ pub(crate) fn read_jobs_state(path: &Path) -> CronJobsState {
}
}
-fn write_jobs_file(path: &Path, jobs: &[AgentCronJob], merge_current: bool) {
+fn write_jobs_file(path: &Path, jobs: &[AgentCronJob], merge_current: bool) -> anyhow::Result<()> {
let current = read_jobs_state(path);
let jobs = if merge_current {
merge_fresh_jobs(current.jobs, jobs.to_vec())
@@ -351,27 +339,35 @@ fn write_jobs_file(path: &Path, jobs: &[AgentCronJob], merge_current: bool) {
jobs,
dispatches: current.dispatches,
},
- );
+ )
}
-pub(crate) fn write_jobs_state(path: &Path, state: &CronJobsState) {
+pub(crate) fn write_jobs_state(path: &Path, state: &CronJobsState) -> anyhow::Result<()> {
if let Some(parent) = path.parent() {
- let _ = std::fs::create_dir_all(parent);
+ std::fs::create_dir_all(parent)?;
}
- let serialized = serde_json::to_string_pretty(state).unwrap_or_default();
- // The one opt-in fsync: losing the atomic rename rolls back to the previous valid file, which
- // cron recovery tolerates (TS `writeJobsState` passes `{ mode: 0o600, fsync: true }`).
- let _ = crate::settings::storage::atomic_write_with(
+ let serialized = serde_json::to_string_pretty(state)?;
+ // Cron state opts into the shared durable write (TS `writeJobsState`
+ // passes `{ mode: 0o600, fsync: true }`).
+ crate::settings::storage::atomic_write_with(
path,
&format!("{serialized}\n"),
crate::settings::storage::AtomicWriteOptions { fsync: true },
- );
+ )
}
#[cfg(test)]
mod tests {
use super::*;
+ #[test]
+ fn jobs_state_write_propagates_physical_failure() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("jobs.json");
+ std::fs::create_dir(&path).unwrap();
+ assert!(write_jobs_state(&path, &CronJobsState::default()).is_err());
+ }
+
/// The cron state write opts in (the TS test pins `{ fsync: true, mode: 0o600 }`)
/// `to_string_pretty(state) + "\n"` bytes.
#[test]
@@ -381,7 +377,7 @@ mod tests {
let state = CronJobsState::default();
let expected = format!("{}\n", serde_json::to_string_pretty(&state).unwrap());
let before = crate::settings::storage::opt_in_fsync_calls();
- write_jobs_state(&path, &state);
+ write_jobs_state(&path, &state).unwrap();
assert_eq!(
crate::settings::storage::opt_in_fsync_calls(),
before + 1,
diff --git a/crates/pa-core/src/mcp/connection_store.rs b/crates/pa-core/src/mcp/connection_store.rs
index f348a5800f..21f01cc3e9 100644
--- a/crates/pa-core/src/mcp/connection_store.rs
+++ b/crates/pa-core/src/mcp/connection_store.rs
@@ -9,7 +9,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
-use crate::settings::storage::atomic_write;
+use crate::settings::storage::{atomic_write_with, AtomicWriteOptions};
/// The connections file version.
const FILE_VERSION: u8 = 1;
@@ -231,7 +231,11 @@ fn write_records(
"version": FILE_VERSION,
"connections": ordered,
});
- atomic_write(path, &serde_json::to_string_pretty(&doc)?)?;
+ atomic_write_with(
+ path,
+ &serde_json::to_string_pretty(&doc)?,
+ AtomicWriteOptions { fsync: true },
+ )?;
Ok(())
}
@@ -262,11 +266,10 @@ pub fn new_pending_record(
mod tests {
use super::*;
- /// Per-call-site served-path oracle (the TS writer passes only `{ mode: 0o600 }`): the
- /// registry write goes through the real `write_records` writer and takes NO fsync
- /// branch.
+ /// Per-call-site served-path oracle: the registry write goes through the real
+ /// `write_records` writer and takes exactly one fsync branch.
#[test]
- fn registry_write_takes_the_ts_default_no_sync() {
+ fn registry_write_takes_exactly_one_fsync() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("connections.json");
let record = McpConnectionRecord {
@@ -293,8 +296,8 @@ mod tests {
write_records(&path, &records).unwrap();
assert_eq!(
crate::settings::storage::opt_in_fsync_calls(),
- before,
- "the TS-default registry write must not sync"
+ before + 1,
+ "the registry write must flush the temp file before the rename"
);
assert_eq!(std::fs::read_to_string(&path).unwrap(), expected);
}
diff --git a/crates/pa-core/src/platform/fs.rs b/crates/pa-core/src/platform/fs.rs
index 9ee89da818..1c71628697 100644
--- a/crates/pa-core/src/platform/fs.rs
+++ b/crates/pa-core/src/platform/fs.rs
@@ -2,6 +2,27 @@
use std::fs::File;
use std::io;
+use std::path::Path;
+
+/// fsync a directory so a completed rename survives a crash on POSIX.
+///
+/// # Errors
+///
+/// Returns an error when the directory cannot be opened or synced.
+#[cfg(unix)]
+pub fn sync_directory(path: &Path) -> io::Result<()> {
+ File::open(path)?.sync_all()
+}
+
+/// Directory fsync is not exposed by the standard library on this platform.
+///
+/// # Errors
+///
+/// This implementation never fails.
+#[cfg(not(unix))]
+pub fn sync_directory(_path: &Path) -> io::Result<()> {
+ Ok(())
+}
/// Flush one file's written bytes with a plain `fsync(2)`; std's
/// `sync_data`/`sync_all` take the `F_FULLFSYNC` barrier on Apple.
diff --git a/crates/pa-core/src/platform/lock_dir.rs b/crates/pa-core/src/platform/lock_dir.rs
index 5cfcd4c872..15c0977a83 100644
--- a/crates/pa-core/src/platform/lock_dir.rs
+++ b/crates/pa-core/src/platform/lock_dir.rs
@@ -1,7 +1,6 @@
//! Cross-process directory locks, byte-compatible with the TS product's
-//! `proper-lockfile` 4.1.2 convention: a lock is an EMPTY DIRECTORY at
-//! `{file}.lock`, staleness is judged from its bumped mtime alone (no pid
-//! or owner file), and a regular file at the lock path is removed on acquisition.
+//! `proper-lockfile` 4.1.2 convention: ordinary locks are empty directories
+//! at `{file}.lock`; harness-state locks add an owner file for safe stale reclaim.
use std::fs;
use std::io;
@@ -171,6 +170,7 @@ mod win32 {
#[derive(Debug)]
pub struct LockDir {
path: PathBuf,
+ owner: Option<String>,
}
impl LockDir {
@@ -191,18 +191,26 @@ impl LockDir {
///
/// [`io::ErrorKind::WouldBlock`] for a fresh foreign lock; other I/O errors as-is.
pub fn acquire(file: &Path, stale_after: Duration) -> io::Result<Self> {
+ Self::acquire_with_owner(file, stale_after, None)
+ }
+
+ fn acquire_with_owner(
+ file: &Path,
+ stale_after: Duration,
+ owner: Option<String>,
+ ) -> io::Result<Self> {
let path = Self::path_for(file);
let stale_after = stale_after.max(MIN_STALE);
- match Self::create(&path) {
- Ok(()) => Ok(LockDir { path }),
+ match Self::create(&path, owner.as_deref()) {
+ Ok(()) => Ok(LockDir { path, owner }),
// Only an existing path is a lock collision; other failures
// (missing parent, permissions) are real errors, never contention.
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
Self::judge_and_reclaim(&path, stale_after)?;
// The judge path removed (or raced away) the incumbent: one
// fresh attempt; a reappearing rival is contention.
- match Self::create(&path) {
- Ok(()) => Ok(LockDir { path }),
+ match Self::create(&path, owner.as_deref()) {
+ Ok(()) => Ok(LockDir { path, owner }),
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
Err(io::Error::new(
io::ErrorKind::WouldBlock,
@@ -216,38 +224,158 @@ impl LockDir {
}
}
- /// The mkdir is the acquisition signal: EEXIST is the only collision.
- #[cfg(unix)]
- fn create(path: &Path) -> io::Result<()> {
- fs::create_dir(path)?;
- let (sec, nanos) = probe_mtime();
- if let Err(error) = set_mtime(path, sec, nanos) {
- // Never leave a lock artifact behind a failed probe.
- let _ = fs::remove_dir(path);
- return Err(error);
+ /// [`Self::acquire`] with a bounded retry: only a fresh foreign lock
+ /// (`WouldBlock`) is retried, sleeping `interval` between attempts;
+ /// any other error returns immediately, and the final `WouldBlock`
+ /// is returned after the last attempt.
+ ///
+ /// # Errors
+ ///
+ /// The last [`io::ErrorKind::WouldBlock`] when all attempts contend;
+ /// other I/O errors as-is.
+ pub fn acquire_retrying(
+ file: &Path,
+ stale_after: Duration,
+ attempts: u32,
+ interval: Duration,
+ ) -> io::Result<Self> {
+ let mut attempt = 0;
+ loop {
+ match Self::acquire(file, stale_after) {
+ Ok(guard) => return Ok(guard),
+ Err(error) if error.kind() != io::ErrorKind::WouldBlock => return Err(error),
+ Err(error) => {
+ attempt += 1;
+ if attempt >= attempts {
+ return Err(error);
+ }
+ std::thread::sleep(interval);
+ }
+ }
+ }
+ }
+
+ /// Acquire a harness-state lock with a PID and per-process token.
+ /// Only a provably dead owner can be reclaimed after `stale_after`.
+ ///
+ /// # Errors
+ ///
+ /// Returns lock contention or an I/O error from acquisition.
+ pub fn acquire_owned_retrying(
+ file: &Path,
+ stale_after: Duration,
+ attempts: u32,
+ interval: Duration,
+ ) -> io::Result<Self> {
+ static PROCESS_TOKEN: std::sync::OnceLock<uuid::Uuid> = std::sync::OnceLock::new();
+ let token = PROCESS_TOKEN.get_or_init(uuid::Uuid::new_v4);
+ let owner = format!("{} {token}.{}", std::process::id(), uuid::Uuid::new_v4());
+ let mut attempt = 0;
+ loop {
+ match Self::acquire_with_owner(file, stale_after, Some(owner.clone())) {
+ Ok(guard) => return Ok(guard),
+ Err(error) if error.kind() != io::ErrorKind::WouldBlock => return Err(error),
+ Err(error) => {
+ attempt += 1;
+ if attempt >= attempts {
+ return Err(error);
+ }
+ std::thread::sleep(interval);
+ }
+ }
+ }
+ }
+
+ /// Check that an owned lock is still held before writing its state file.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the owner file no longer matches this guard.
+ pub fn ensure_owned(&self) -> io::Result<()> {
+ if self
+ .owner
+ .as_ref()
+ .is_some_and(|owner| !Self::owner_matches(&self.path, owner))
+ {
+ return Err(io::Error::other(format!(
+ "harness state lock lost: {}",
+ self.path.display()
+ )));
}
Ok(())
}
- /// The mkdir is the acquisition signal; the mtime probe keeps staleness
- /// meaningful on NTFS (directory mtimes otherwise sit on the second).
- #[cfg(windows)]
- fn create(path: &Path) -> io::Result<()> {
+ fn owner_matches(path: &Path, owner: &str) -> bool {
+ fs::read_to_string(path.join("owner")).is_ok_and(|recorded| recorded.trim() == owner)
+ }
+
+ fn owner_dead(recorded: &str) -> bool {
+ let Some((pid, token)) = recorded.trim().split_once(' ') else {
+ return true;
+ };
+ if token.is_empty() {
+ return true;
+ }
+ let Ok(pid) = pid.parse::<u32>() else {
+ return true;
+ };
+ if pid == 0 || pid > i32::MAX as u32 {
+ return true;
+ }
+ #[cfg(unix)]
+ {
+ let result = unsafe { libc::kill(pid as i32, 0) };
+ result != 0 && io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH)
+ }
+ #[cfg(windows)]
+ {
+ matches!(
+ pa_types::platform::process::is_process_alive(pid),
+ Ok(false)
+ )
+ }
+ #[cfg(not(any(unix, windows)))]
+ {
+ false
+ }
+ }
+
+ /// The mkdir is the acquisition signal: EEXIST is the only collision.
+ #[cfg(any(unix, windows))]
+ fn create(path: &Path, owner: Option<&str>) -> io::Result<()> {
+ #[cfg(unix)]
+ use std::os::unix::fs::PermissionsExt;
fs::create_dir(path)?;
- let (sec, nanos) = probe_mtime();
- if let Err(error) = set_mtime(path, sec, nanos) {
- // Never leave a lock artifact behind a failed probe.
+ let result = (|| {
+ if let Some(owner) = owner {
+ #[cfg(unix)]
+ {
+ fs::set_permissions(path, fs::Permissions::from_mode(0o700))?;
+ }
+ fs::write(path.join("owner"), format!("{owner}\n"))?;
+ #[cfg(unix)]
+ fs::set_permissions(path.join("owner"), fs::Permissions::from_mode(0o600))?;
+ }
+ let (sec, nanos) = probe_mtime();
+ set_mtime(path, sec, nanos)
+ })();
+ if result.is_err() {
+ let _ = fs::remove_file(path.join("owner"));
let _ = fs::remove_dir(path);
- return Err(error);
}
- Ok(())
+ result
}
#[cfg(not(any(unix, windows)))]
- fn create(path: &Path) -> io::Result<()> {
- // No mtime probe on this platform: staleness is judged from the
- // filesystem's own directory mtime.
- fs::create_dir(path)
+ fn create(path: &Path, owner: Option<&str>) -> io::Result<()> {
+ fs::create_dir(path)?;
+ if let Some(owner) = owner {
+ if let Err(error) = fs::write(path.join("owner"), format!("{owner}\n")) {
+ let _ = fs::remove_dir(path);
+ return Err(error);
+ }
+ }
+ Ok(())
}
/// Decide the fate of an incumbent at `path`. Returns only when the
@@ -286,6 +414,22 @@ impl LockDir {
.duration_since(modified)
.unwrap_or_default();
if age > stale_after {
+ let owner_path = path.join("owner");
+ let recorded = fs::read_to_string(&owner_path).ok();
+ if recorded
+ .as_deref()
+ .is_some_and(|owner| !Self::owner_dead(owner))
+ {
+ return Err(io::Error::new(
+ io::ErrorKind::WouldBlock,
+ format!("Lock file is already being held: {}", path.display()),
+ ));
+ }
+ match fs::remove_file(&owner_path) {
+ Ok(()) => {}
+ Err(error) if error.kind() == io::ErrorKind::NotFound => {}
+ Err(error) => return Err(error),
+ }
// Stale: remove and let the caller retry.
match fs::remove_dir(path) {
Ok(()) => return Ok(()),
@@ -319,6 +463,15 @@ impl LockDir {
/// else already reclaimed it (the TS release tolerates ENOENT); other
/// failures go to the trace log (`Drop` cannot propagate).
pub fn release(&self) {
+ if let Some(owner) = &self.owner {
+ if !Self::owner_matches(&self.path, owner) {
+ return;
+ }
+ if let Err(error) = fs::remove_file(self.path.join("owner")) {
+ tracing::warn!("failed to release lock {}: {error}", self.path.display());
+ return;
+ }
+ }
if let Err(error) = fs::remove_dir(&self.path) {
if error.kind() != io::ErrorKind::NotFound {
tracing::warn!("failed to release lock {}: {error}", self.path.display());
@@ -405,6 +558,55 @@ mod tests {
assert!(!stale.exists());
}
+ #[cfg(unix)]
+ #[test]
+ fn live_owned_lock_cannot_be_stolen_after_stale_age() {
+ let dir = tempfile::tempdir().unwrap();
+ let file = dir.path().join("harness_state.json");
+ let guard =
+ LockDir::acquire_owned_retrying(&file, Duration::from_secs(10), 1, MIN_STALE).unwrap();
+ set_mtime(&guard.path, 1, 0).unwrap();
+ let error = LockDir::acquire_owned_retrying(&file, Duration::from_secs(10), 1, MIN_STALE)
+ .unwrap_err();
+ assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
+ guard.ensure_owned().unwrap();
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn dead_owned_lock_is_reclaimed_and_old_guard_cannot_remove_successor() {
+ let dir = tempfile::tempdir().unwrap();
+ let file = dir.path().join("harness_state.json");
+ let old =
+ LockDir::acquire_owned_retrying(&file, Duration::from_secs(10), 1, MIN_STALE).unwrap();
+ let mut child = std::process::Command::new("true").spawn().unwrap();
+ let dead_pid = child.id();
+ child.wait().unwrap();
+ fs::write(old.path.join("owner"), format!("{dead_pid} dead-token\n")).unwrap();
+ set_mtime(&old.path, 1, 0).unwrap();
+ let next =
+ LockDir::acquire_owned_retrying(&file, Duration::from_secs(10), 1, MIN_STALE).unwrap();
+ assert!(old.ensure_owned().is_err());
+ drop(old);
+ next.ensure_owned().unwrap();
+ drop(next);
+ assert!(!LockDir::path_for(&file).exists());
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn unparseable_owned_lock_is_reclaimed_by_age() {
+ let dir = tempfile::tempdir().unwrap();
+ let file = dir.path().join("harness_state.json");
+ let path = LockDir::path_for(&file);
+ fs::create_dir(&path).unwrap();
+ fs::write(path.join("owner"), [0xff]).unwrap();
+ set_mtime(&path, 1, 0).unwrap();
+ let next =
+ LockDir::acquire_owned_retrying(&file, Duration::from_secs(10), 1, MIN_STALE).unwrap();
+ next.ensure_owned().unwrap();
+ }
+
#[test]
fn legacy_lock_file_is_removed_not_choked_on() {
// A pre-compat Rust build left flock FILES at the lock path (the TS
diff --git a/crates/pa-core/src/refinement/mod.rs b/crates/pa-core/src/refinement/mod.rs
index 025807fb04..309a1f00b7 100644
--- a/crates/pa-core/src/refinement/mod.rs
+++ b/crates/pa-core/src/refinement/mod.rs
@@ -283,10 +283,40 @@ pub fn save_harness_state(
std::fs::create_dir_all(harness_state_dir)?;
let state_path = get_harness_state_path(harness_state_dir);
let content = format!("{}\n", serde_json::to_string_pretty(state)?);
- crate::settings::storage::atomic_write(&state_path, &content)?;
+ crate::settings::storage::atomic_write_with(
+ &state_path,
+ &content,
+ crate::settings::storage::AtomicWriteOptions { fsync: true },
+ )?;
Ok(state_path)
}
+/// Locked read-modify-write of the harness state file: both writer sides
+/// (the kernel and refine) take `{file}.lock` around the reload and the save.
+///
+/// # Errors
+///
+/// Error when the directory cannot be created, the state lock cannot be acquired, or the atomic write fails.
+pub fn update_harness_state<R>(
+ harness_state_dir: &Path,
+ scope: HarnessScope,
+ update: impl FnOnce(&mut HarnessState) -> R,
+) -> anyhow::Result<(R, PathBuf)> {
+ std::fs::create_dir_all(harness_state_dir)?;
+ let state_path = get_harness_state_path(harness_state_dir);
+ let lock = crate::platform::lock_dir::LockDir::acquire_owned_retrying(
+ &state_path,
+ std::time::Duration::from_secs(10),
+ 50,
+ std::time::Duration::from_millis(20),
+ )?;
+ let mut state = load_harness_state(harness_state_dir, scope);
+ let result = update(&mut state);
+ lock.ensure_owned()?;
+ let written = save_harness_state(harness_state_dir, &state)?;
+ Ok((result, written))
+}
+
#[must_use]
pub fn get_refinement_history_path(harness_state_dir: &Path) -> PathBuf {
harness_state_dir.join(REFINEMENT_HISTORY_FILE_NAME)
@@ -636,11 +666,8 @@ mod tests {
);
}
- /// Served-path oracle (refinement.ts:404 passes only `{ mode }` — the
- /// measured signal of record 20260928-172400): the save takes NO fsync
- /// branch, landing exactly `to_string_pretty(state) + "\n"` bytes.
#[test]
- fn harness_save_takes_the_ts_default_no_sync() {
+ fn harness_save_takes_exactly_one_fsync() {
let tmp = tempfile::tempdir().unwrap();
let dir = get_global_harness_state_dir(tmp.path());
let mut state = empty_harness_state();
@@ -657,8 +684,8 @@ mod tests {
let written = save_harness_state(&dir, &state).unwrap();
assert_eq!(
crate::settings::storage::opt_in_fsync_calls(),
- before,
- "the TS-default harness save must not sync"
+ before + 1,
+ "the harness save must flush the temp file before the rename"
);
assert_eq!(written, get_harness_state_path(&dir));
assert_eq!(
@@ -667,6 +694,33 @@ mod tests {
);
}
+ #[test]
+ fn concurrent_update_harness_state_writes_all_land() {
+ let tmp = tempfile::tempdir().unwrap();
+ let dir = get_global_harness_state_dir(tmp.path());
+ std::thread::scope(|scope| {
+ for worker in 0..8u32 {
+ let dir = &dir;
+ scope.spawn(move || {
+ for i in 0..10u32 {
+ let id = format!("m-{worker}-{i}");
+ update_harness_state(dir, HarnessScope::Global, |state| {
+ let memories = state.entries.get_mut(&RefinementKind::Memory).unwrap();
+ memories.insert(
+ id.clone(),
+ entry(&id, RefinementKind::Memory, HarnessScope::Global, "a fact"),
+ );
+ memories.len()
+ })
+ .unwrap();
+ }
+ });
+ }
+ });
+ let loaded = load_harness_state(&dir, HarnessScope::Global);
+ assert_eq!(loaded.entries[&RefinementKind::Memory].len(), 80);
+ }
+
#[test]
fn merge_prefixed_local_conflicts() {
let mut global = empty_harness_state();
diff --git a/crates/pa-core/src/session_engine/host_requests.rs b/crates/pa-core/src/session_engine/host_requests.rs
index 68981f02bc..8c1ec24418 100644
--- a/crates/pa-core/src/session_engine/host_requests.rs
+++ b/crates/pa-core/src/session_engine/host_requests.rs
@@ -306,7 +306,7 @@ pub fn handle_rlm_heartbeat_host_request(
let Some(id) = record.get("id").and_then(Value::as_str) else {
anyhow::bail!("rlm_heartbeat.delete id must be a string");
};
- let heartbeat = store.delete_rlm_heartbeat(active_session_id, id, now);
+ let heartbeat = store.delete_rlm_heartbeat(active_session_id, id, now)?;
Ok(RlmHeartbeatHostOutcome {
response: json!({
"heartbeat": heartbeat
diff --git a/crates/pa-core/src/session_engine/refine.rs b/crates/pa-core/src/session_engine/refine.rs
index 2c8a6e245e..90b8747908 100644
--- a/crates/pa-core/src/session_engine/refine.rs
+++ b/crates/pa-core/src/session_engine/refine.rs
@@ -14,7 +14,7 @@ use crate::refinement::executor::{
};
use crate::refinement::{
append_global_refinement, format_refinement_notice_body, load_global_refinement_history,
- load_harness_state, merge_harness_states, save_harness_state, HarnessScope, RefinementResult,
+ load_harness_state, merge_harness_states, update_harness_state, HarnessScope, RefinementResult,
};
use crate::session::manager::SessionManager;
@@ -374,7 +374,6 @@ pub async fn execute_refinement_with_rows(
HarnessScope::Global => global_harness_dir.to_path_buf(),
HarnessScope::Local => local_harness_dir.clone(),
};
- let mut state = load_harness_state(&target_dir, target_scope);
// The factory opt-in resolves HERE — immediately before the apply,
// after the planning request — so a setting that changed during the
// request (`/factory off` mid-plan) decides, not a snapshot captured
@@ -393,16 +392,20 @@ pub async fn execute_refinement_with_rows(
}
None => false,
};
- let mut result = apply_refinement_plan(
- &mut state,
- plan,
- &core_options,
- Some(baseline_state),
- factory_enabled,
- );
- result.harness_state_path = save_harness_state(&target_dir, &state)?
- .to_string_lossy()
- .to_string();
+ let (mut result, state_path) = tokio::task::spawn_blocking(move || {
+ update_harness_state(&target_dir, target_scope, |state| {
+ apply_refinement_plan(
+ state,
+ plan,
+ &core_options,
+ Some(baseline_state),
+ factory_enabled,
+ )
+ })
+ })
+ .await
+ .map_err(|error| anyhow::anyhow!("refinement harness update task failed: {error}"))??;
+ result.harness_state_path = state_path.to_string_lossy().to_string();
if target_scope == HarnessScope::Global {
append_global_refinement(global_harness_dir, &result)?;
}
diff --git a/crates/pa-core/src/settings/storage.rs b/crates/pa-core/src/settings/storage.rs
index 2270a8dac7..ebb251705f 100644
--- a/crates/pa-core/src/settings/storage.rs
+++ b/crates/pa-core/src/settings/storage.rs
@@ -249,7 +249,7 @@ impl SettingsStorage for FileSettingsStorage {
}
}
if let Some(content) = next {
- atomic_write(path, &content)?;
+ atomic_write_with(path, &content, AtomicWriteOptions { fsync: true })?;
}
}
drop(held);
@@ -302,6 +302,9 @@ pub fn atomic_write_with(path: &Path, content: &str, options: AtomicWriteOptions
}
}
crate::platform::rename_onto(&temp, path)?;
+ if options.fsync {
+ crate::platform::fs::sync_directory(path.parent().unwrap_or_else(|| Path::new(".")))?;
+ }
Ok(())
}
@@ -399,7 +402,7 @@ mod tests {
}
#[test]
- fn settings_write_takes_the_ts_default_no_sync() {
+ fn settings_write_takes_exactly_one_fsync() {
let dir = tempfile::tempdir().unwrap();
let storage = FileSettingsStorage::new(dir.path().join("cwd"), dir.path().join("agent"));
let document = "{ \"defaultProvider\": \"prime-inference\" }";
@@ -407,7 +410,11 @@ mod tests {
storage
.with_lock(SettingsScope::Global, &mut |_| Some(document.to_string()))
.unwrap();
- assert_eq!(opt_in_fsync_calls(), before);
+ assert_eq!(
+ opt_in_fsync_calls(),
+ before + 1,
+ "the settings write must flush the temp file before the rename"
+ );
let path = dir.path().join("agent").join("settings.json");
assert_eq!(fs::read_to_string(&path).unwrap(), document);
}
diff --git a/crates/pa-core/src/update/download.rs b/crates/pa-core/src/update/download.rs
index 72110126fe..6d63033bfb 100644
--- a/crates/pa-core/src/update/download.rs
+++ b/crates/pa-core/src/update/download.rs
@@ -306,7 +306,7 @@ fn update_install_source(release_dir: &Path, install_source: &str) -> Result<()>
}
std::fs::write(&path, install_source).with_context(|| format!("write {}", path.display()))?;
std::fs::File::open(&path)?.sync_all()?;
- super::install::sync_directory(release_dir)
+ Ok(super::install::sync_directory(release_dir)?)
}
/// Copy or extract the payload into staging scratch, check the release
@@ -395,7 +395,7 @@ fn sync_release_tree(root: &Path) -> Result<()> {
for directory in directories.iter().rev() {
super::install::sync_directory(directory)?;
}
- super::install::sync_directory(root)
+ Ok(super::install::sync_directory(root)?)
}
/// Validate a staged payload, write the installer metadata, and rename it into
diff --git a/crates/pa-core/src/update/install.rs b/crates/pa-core/src/update/install.rs
index 477b00dc20..d33abd5e82 100644
--- a/crates/pa-core/src/update/install.rs
+++ b/crates/pa-core/src/update/install.rs
@@ -204,18 +204,7 @@ pub fn install_source_is_valid(source: &str) -> bool {
protocol == "https" || protocol == "http"
}
-/// fsync a directory so its entries survive a crash (staged releases and
-/// the launcher repoint must never reference an unwritten inode; a
-/// directory fd opened read-only accepts `sync_all` on POSIX platforms).
-///
-/// # Errors
-///
-/// Returns an error when the directory cannot be opened or its `sync_all` fails.
-pub fn sync_directory(path: &Path) -> Result<()> {
- let dir = std::fs::File::open(path)?;
- dir.sync_all()
- .with_context(|| format!("fsync {}", path.display()))
-}
+pub use crate::platform::fs::sync_directory;
/// The parsed shape of a launcher link target.
struct ReleaseLink {
diff --git a/crates/pa-daemon/src/agent_engine/goalcore.rs b/crates/pa-daemon/src/agent_engine/goalcore.rs
index 660bb9b855..31c2300e18 100644
--- a/crates/pa-daemon/src/agent_engine/goalcore.rs
+++ b/crates/pa-daemon/src/agent_engine/goalcore.rs
@@ -137,7 +137,9 @@ impl AgentSessionEngine {
{
continue;
}
- let _ = wiring.store.cancel(&job.id, crate::util::now_ms());
+ if let Err(error) = wiring.store.cancel(&job.id, crate::util::now_ms()) {
+ eprintln!("failed to cancel goal wake job: {error}");
+ }
}
}
diff --git a/crates/pa-daemon/src/agent_engine/turn/quota.rs b/crates/pa-daemon/src/agent_engine/turn/quota.rs
index 2a9acbb0c5..83ae59f861 100644
--- a/crates/pa-daemon/src/agent_engine/turn/quota.rs
+++ b/crates/pa-daemon/src/agent_engine/turn/quota.rs
@@ -75,7 +75,9 @@ impl AgentSessionEngine {
.iter()
.any(|job| job.id == job_id && job.status == pa_core::cron::JobStatus::Active);
if matches_job {
- let _ = wiring.store.cancel(job_id, crate::util::now_ms());
+ if let Err(error) = wiring.store.cancel(job_id, crate::util::now_ms()) {
+ eprintln!("failed to cancel quota resume job: {error}");
+ }
}
}
diff --git a/crates/pa-daemon/src/branch_navigation.rs b/crates/pa-daemon/src/branch_navigation.rs
index a90c62e80e..a873a5c8a3 100644
--- a/crates/pa-daemon/src/branch_navigation.rs
+++ b/crates/pa-daemon/src/branch_navigation.rs
@@ -566,7 +566,9 @@ impl Worker {
// source branch.
self.refresh_replaced_session_state().await;
self.reseed_service_tier_for_replacement();
- self.bind_scheduled_jobs().await;
+ if let Err(error) = self.bind_scheduled_jobs().await {
+ return response_failure(None, "fork", &error.to_string(), None);
+ }
self.prewarm_replacement_session();
// The fork swap is a whole-session replacement too: the fresh summary ships
// immediately.
diff --git a/crates/pa-daemon/src/scheduled_jobs.rs b/crates/pa-daemon/src/scheduled_jobs.rs
index c280f652b2..4e009717d6 100644
--- a/crates/pa-daemon/src/scheduled_jobs.rs
+++ b/crates/pa-daemon/src/scheduled_jobs.rs
@@ -80,7 +80,7 @@ impl QueueHooks {
/// The failed-runnable cancel: the store cancels the dead session's whole
/// job set by file, so the artifact never re-fires.
- fn cancel_jobs_for_dead_target(&self, job: &AgentCronJob) {
+ fn cancel_jobs_for_dead_target(&self, job: &AgentCronJob) -> anyhow::Result<()> {
self.store.cancel_jobs_for_session(
&CancelJobsFilter {
active_session_id: None,
@@ -88,7 +88,8 @@ impl QueueHooks {
session_file: Some(job.session_file.clone()),
},
crate::util::now_ms(),
- );
+ )?;
+ Ok(())
}
}
@@ -97,7 +98,7 @@ impl AgentCronSchedulerHooks for QueueHooks {
// A persisted job whose target is no longer live (killed — `archived` — or deleted)
// cancels the session's jobs and skips: a fire can never revive a stopped session.
if Self::persisted_target_gone(job) {
- self.cancel_jobs_for_dead_target(job);
+ self.cancel_jobs_for_dead_target(job)?;
return Ok(Some("skipped"));
}
let activity = self.activity();
@@ -260,19 +261,19 @@ impl ScheduledJobs {
&self,
binding: SessionBinding,
artifact_dir: Option<PathBuf>,
- ) {
+ ) -> anyhow::Result<()> {
if let Some(dir) = artifact_dir {
- let _ = std::fs::create_dir_all(&dir);
+ std::fs::create_dir_all(&dir)?;
self.store
.register_session_artifact(&binding.session_id, &dir);
}
if !binding.session_file.is_empty() {
- self.store.rebind_session_jobs(&binding);
+ self.store.rebind_session_jobs(&binding)?;
}
let mut guard = self.scheduler.lock().await;
if let Some(scheduler) = guard.as_ref() {
scheduler.wake().await;
- return;
+ return Ok(());
}
let scheduler = Arc::new(AgentCronScheduler::new(
Arc::clone(&self.store),
@@ -280,6 +281,7 @@ impl ScheduledJobs {
));
scheduler.start().await;
*guard = Some(scheduler);
+ Ok(())
}
/// Re-arm the timer after a catalog mutation (TS `cronScheduler.wake`).
@@ -374,7 +376,7 @@ impl Worker {
/// The killed close's cancel: the session's whole job set cancels by any of
/// its three identities — durably, so the session's own heartbeats can
/// never revive it.
- pub(crate) async fn cancel_session_scheduled_jobs(&self) {
+ pub(crate) async fn cancel_session_scheduled_jobs(&self) -> anyhow::Result<()> {
let (active_session_id, session_id, session_file) = {
let core = self
.core
@@ -382,7 +384,7 @@ impl Worker {
.unwrap_or_else(std::sync::PoisonError::into_inner);
self.bind_store_artifact(&core);
let Some(store) = core.store.as_ref() else {
- return;
+ return Ok(());
};
(
core.active_session_id.clone(),
@@ -397,18 +399,19 @@ impl Worker {
session_file: Some(session_file),
},
crate::util::now_ms(),
- );
+ )?;
for job in &cancelled {
self.scheduled.remove_queued_heartbeat_follow_up(job);
}
if !cancelled.is_empty() {
self.scheduled.wake().await;
}
+ Ok(())
}
/// Only a subagent's RLM heartbeat jobs cancel here (the plain cron jobs
/// survive the replacement); a top-level session cancels nothing.
- pub(crate) async fn cancel_session_rlm_heartbeats(&self) {
+ pub(crate) async fn cancel_session_rlm_heartbeats(&self) -> anyhow::Result<()> {
let (is_subagent, active_session_id) = {
let core = self
.core
@@ -421,18 +424,19 @@ impl Worker {
)
};
if !is_subagent {
- return;
+ return Ok(());
}
let cancelled = self
.scheduled
.store()
- .cancel_rlm_heartbeats_for_session(&active_session_id, crate::util::now_ms());
+ .cancel_rlm_heartbeats_for_session(&active_session_id, crate::util::now_ms())?;
for job in &cancelled {
self.scheduled.remove_queued_heartbeat_follow_up(job);
}
if !cancelled.is_empty() {
self.scheduled.wake().await;
}
+ Ok(())
}
/// The saved-session delete's cancel: cancel the deleted file's whole job
@@ -458,14 +462,16 @@ impl Worker {
self.scheduled
.store()
.register_session_artifact(session_id, &dir);
- self.scheduled.store().cancel_jobs_for_session(
+ if let Err(error) = self.scheduled.store().cancel_jobs_for_session(
&pa_core::cron::store::CancelJobsFilter {
active_session_id: None,
session_id: None,
session_file: Some(session_file.to_string_lossy().to_string()),
},
crate::util::now_ms(),
- );
+ ) {
+ eprintln!("failed to cancel deleted session jobs: {error}");
+ }
}
}
diff --git a/crates/pa-daemon/src/scheduled_jobs/arms.rs b/crates/pa-daemon/src/scheduled_jobs/arms.rs
index 381385a0cc..82382a0e5e 100644
--- a/crates/pa-daemon/src/scheduled_jobs/arms.rs
+++ b/crates/pa-daemon/src/scheduled_jobs/arms.rs
@@ -218,7 +218,7 @@ impl Worker {
.store()
.cancel(&job_id, crate::util::now_ms())
{
- Some(job) => {
+ Ok(Some(job)) => {
self.scheduled.remove_queued_heartbeat_follow_up(&job);
self.scheduled.wake().await;
response_success(
@@ -227,12 +227,13 @@ impl Worker {
Some(json!({ "job": serde_json::to_value(&job).unwrap_or(Value::Null) })),
)
}
- None => response_failure(
+ Ok(None) => response_failure(
None,
"cron_cancel",
&format!("No cron job found: {job_id}"),
None,
),
+ Err(error) => response_failure(None, "cron_cancel", &error.to_string(), None),
}
}
@@ -364,20 +365,20 @@ impl Worker {
self.bind_store_artifact(&core);
}
let outcome = match payload.get("action").and_then(Value::as_str) {
- Some("pause") => Ok(self
+ Some("pause") => self
.scheduled
.store()
- .pause_heartbeat(&active_session_id, now)),
+ .pause_heartbeat(&active_session_id, now),
Some("resume") => self
.scheduled
.store()
.resume_heartbeat(&active_session_id, now),
// TS `updateHeartbeatForState`: anything but pause/resume clears the
// heartbeat.
- _ => Ok(self
+ _ => self
.scheduled
.store()
- .clear_heartbeat(&active_session_id, now)),
+ .clear_heartbeat(&active_session_id, now),
};
let outcome = match outcome {
Ok(job) => job,
diff --git a/crates/pa-daemon/src/scheduling_catalog.rs b/crates/pa-daemon/src/scheduling_catalog.rs
index 7d38f50d21..dbd0479dc9 100644
--- a/crates/pa-daemon/src/scheduling_catalog.rs
+++ b/crates/pa-daemon/src/scheduling_catalog.rs
@@ -777,16 +777,32 @@ impl Supervisor {
.find(|passive| passive.job.id == *job_id);
if let Some(passive) = passive {
let store = Self::passive_job_store(&passive.info);
- if let Some(job) = store.cancel(job_id, crate::util::now_ms()) {
- self.broadcast_heartbeats_changed();
- return (
- vec![response_line(&response_success(
- Some(command_id),
- type_name,
- Some(json!({ "job": serde_json::to_value(&job).unwrap_or(Value::Null) })),
- ))],
- false,
- );
+ match store.cancel(job_id, crate::util::now_ms()) {
+ Ok(Some(job)) => {
+ self.broadcast_heartbeats_changed();
+ return (
+ vec![response_line(&response_success(
+ Some(command_id),
+ type_name,
+ Some(
+ json!({ "job": serde_json::to_value(&job).unwrap_or(Value::Null) }),
+ ),
+ ))],
+ false,
+ );
+ }
+ Ok(None) => {}
+ Err(error) => {
+ return (
+ vec![response_line(&response_failure(
+ Some(command_id),
+ type_name,
+ &error.to_string(),
+ None,
+ ))],
+ false,
+ );
+ }
}
}
(
diff --git a/crates/pa-daemon/src/session_navigation.rs b/crates/pa-daemon/src/session_navigation.rs
index 60359d3ed5..c9bc3675a5 100644
--- a/crates/pa-daemon/src/session_navigation.rs
+++ b/crates/pa-daemon/src/session_navigation.rs
@@ -316,7 +316,9 @@ impl Worker {
Ok(()) => {
self.refresh_replaced_session_state().await;
self.reseed_service_tier_for_replacement();
- self.bind_scheduled_jobs().await;
+ if let Err(error) = self.bind_scheduled_jobs().await {
+ return response_failure(None, command, &error.to_string(), None);
+ }
self.prewarm_replacement_session();
// The replacement never pushed a roster delta, so the subscribed
// surfaces kept the PREVIOUS session's numbers.
diff --git a/crates/pa-daemon/src/stop_cleanup.rs b/crates/pa-daemon/src/stop_cleanup.rs
index ac731bf54d..bca4225cda 100644
--- a/crates/pa-daemon/src/stop_cleanup.rs
+++ b/crates/pa-daemon/src/stop_cleanup.rs
@@ -100,7 +100,7 @@ pub(crate) fn cancel_scheduled_jobs_for_tree(
sessions_dir: &Path,
root_session_file: &Path,
live_files: &HashSet<String>,
-) -> usize {
+) -> Result<usize> {
let members = session_tree(agent_dir, sessions_dir, root_session_file);
if members.iter().any(|member| {
live_files.contains(
@@ -109,7 +109,7 @@ pub(crate) fn cancel_scheduled_jobs_for_tree(
.to_string(),
)
}) {
- return 0;
+ return Ok(0);
}
let store = AgentCronJobStore::for_session_artifacts();
let mut registered = false;
@@ -127,7 +127,7 @@ pub(crate) fn cancel_scheduled_jobs_for_tree(
registered = true;
}
if !registered {
- return 0;
+ return Ok(0);
}
let now = crate::util::now_ms();
let mut cancelled = 0;
@@ -140,10 +140,10 @@ pub(crate) fn cancel_scheduled_jobs_for_tree(
session_file: Some(member.session_file.to_string_lossy().to_string()),
},
now,
- )
+ )?
.len();
}
- cancelled
+ Ok(cancelled)
}
/// The killed session's `archived` state belt: when the worker died before its close
@@ -176,6 +176,10 @@ pub(crate) fn finalize_archived_stop(
) -> (usize, Option<anyhow::Error>) {
let cancelled =
cancel_scheduled_jobs_for_tree(agent_dir, sessions_dir, root_session_file, live_files);
+ let (cancelled, cancel_error) = match cancelled {
+ Ok(count) => (count, None),
+ Err(error) => (0, Some(error)),
+ };
// A live worker covering the tree owns its stores again — including the session
// file's state — so the belt skips covered trees like the cancel does.
let covered = session_tree(agent_dir, sessions_dir, root_session_file)
@@ -188,10 +192,10 @@ pub(crate) fn finalize_archived_stop(
)
});
if covered {
- return (cancelled, None);
+ return (cancelled, cancel_error);
}
let archive_error = ensure_archived_state(root_session_file).err();
- (cancelled, archive_error)
+ (cancelled, cancel_error.or(archive_error))
}
/// The kill route's deleted-child finalize (the `rlmLedgerDelete` marker):
@@ -474,6 +478,13 @@ impl Supervisor {
Path::new(&root_session_file),
&live,
);
+ let cancelled = match cancelled {
+ Ok(count) => count,
+ Err(error) => {
+ self.log_line(&format!("owned stop: cron cancel failed: {error}"));
+ return;
+ }
+ };
if cancelled > 0 {
self.log_line(&format!(
"owned stop: cancelled {cancelled} scheduled job(s) of {root_session_file}"
diff --git a/crates/pa-daemon/src/worker/commands.rs b/crates/pa-daemon/src/worker/commands.rs
index ac794c0193..0bc684187b 100644
--- a/crates/pa-daemon/src/worker/commands.rs
+++ b/crates/pa-daemon/src/worker/commands.rs
@@ -567,10 +567,13 @@ impl Worker {
// cron jobs but cancels the subagent's RLM heartbeats; `shutdown` keeps them
// all. The store cancel is durable, so the stopped session's own heartbeats
// can never revive it.
- match reason {
+ let cancel_result = match reason {
KillCloseReason::Killed => self.cancel_session_scheduled_jobs().await,
KillCloseReason::Replaced => self.cancel_session_rlm_heartbeats().await,
- KillCloseReason::Shutdown => {}
+ KillCloseReason::Shutdown => Ok(()),
+ };
+ if let Err(error) = cancel_result {
+ return response_failure(None, "kill", &error.to_string(), None);
}
// The close cascades to the resident children with the SAME reason before
// the session's own archive and dispose; a close failure is swallowed.
diff --git a/crates/pa-daemon/src/worker/create.rs b/crates/pa-daemon/src/worker/create.rs
index 53ed743edd..493a9b5fd1 100644
--- a/crates/pa-daemon/src/worker/create.rs
+++ b/crates/pa-daemon/src/worker/create.rs
@@ -586,7 +586,9 @@ impl Worker {
}
// Bind the schedule catalog onto the session (artifact partition,
// job rebind, scheduler start) — TS `rebindCronJobsToState`.
- self.bind_scheduled_jobs().await;
+ if let Err(error) = self.bind_scheduled_jobs().await {
+ return response_failure(None, "create", &error.to_string(), None);
+ }
// Recovery journal writes must not happen while holding the core
// lock: record_recovery locks the core to read the store.
let _ = self.record_recovery(true, "create");
diff --git a/crates/pa-daemon/src/worker/lifecycle.rs b/crates/pa-daemon/src/worker/lifecycle.rs
index b88f4eb868..8fd260f184 100644
--- a/crates/pa-daemon/src/worker/lifecycle.rs
+++ b/crates/pa-daemon/src/worker/lifecycle.rs
@@ -298,7 +298,7 @@ impl Worker {
/// Bind the live session's schedule catalog: register the artifact
/// partition, rebind the stored jobs onto the live ids, and start (or
/// wake) the scheduler. Runs at create and after every replacement swap.
- pub(crate) async fn bind_scheduled_jobs(&self) {
+ pub(crate) async fn bind_scheduled_jobs(&self) -> anyhow::Result<()> {
let binding = {
let core = self
.core
@@ -307,8 +307,9 @@ impl Worker {
crate::scheduled_jobs::live_binding(&core)
};
if let Some((binding, artifact_dir)) = binding {
- self.scheduled.bind_session(binding, artifact_dir).await;
+ self.scheduled.bind_session(binding, artifact_dir).await?;
}
+ Ok(())
}
/// Clear the queued-input suspension and wake the turn runner so parked
diff --git a/crates/pa-models/src/cache.rs b/crates/pa-models/src/cache.rs
index dd55378963..b08df0d749 100644
--- a/crates/pa-models/src/cache.rs
+++ b/crates/pa-models/src/cache.rs
@@ -472,7 +472,6 @@ fn write_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
let mut file = write()?;
file.write_all(bytes)?;
file.flush()?;
- file.sync_all()?;
std::fs::rename(&temp, path)
}
diff --git a/prime-agent-runtime/src/rlm/harness.py b/prime-agent-runtime/src/rlm/harness.py
index 4d399da3a6..9b43b8dd70 100644
--- a/prime-agent-runtime/src/rlm/harness.py
+++ b/prime-agent-runtime/src/rlm/harness.py
@@ -9,13 +9,17 @@ Execution still belongs to Prime Agent's TypeScript host and the existing
from __future__ import annotations
+import contextlib
import copy
import json
import math
import os
import re
import stat
+import sys
+import time
import unicodedata
+from collections.abc import Iterator
from dataclasses import asdict, dataclass, field, fields
from datetime import datetime, timezone
from pathlib import Path
@@ -24,6 +28,9 @@ from typing import Any, Literal
from .factory import require_factory_enabled, validate_factory_spec
+if sys.platform == "darwin":
+ import fcntl
+
HarnessKind = Literal["prompt", "memory", "skill", "subagent", "factory"]
HarnessScope = Literal["local", "global"]
@@ -31,6 +38,10 @@ _DEFAULT_FILE_NAME = "harness_state.json"
_DEFAULT_HARNESS_DIR_NAME = "harness"
_KINDS: tuple[HarnessKind, ...] = ("prompt", "memory", "skill", "subagent", "factory")
_state_cache: dict[tuple[Path, HarnessScope], "HarnessState"] = {}
+_LOCK_STALE_AFTER = 10.0
+_LOCK_ATTEMPTS = 50
+_LOCK_RETRY_INTERVAL = 0.02
+_PROCESS_TOKEN = uuid4().hex
def _now() -> str:
@@ -406,6 +417,7 @@ class HarnessState:
# mtime of the file as of the last load/save, used to detect out-of-process
# writes (e.g. the host `/refine` command) and avoid clobbering them.
self._loaded_mtime: int | None = None
+ self._held_lock: tuple[Path, str] | None = None
self.load()
def _ensure_local_writable(self) -> None:
@@ -432,6 +444,83 @@ class HarnessState:
if self._disk_mtime() != self._loaded_mtime:
self.load()
+ @staticmethod
+ def _owner_dead(recorded: str) -> bool:
+ parts = recorded.strip().split(" ")
+ if len(parts) != 2 or not parts[1]:
+ return True
+ try:
+ pid = int(parts[0])
+ except ValueError:
+ return True
+ if pid <= 0:
+ return True
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return True
+ except (PermissionError, OSError):
+ return False
+ return False
+
+ @staticmethod
+ def _owner_matches(lock_path: Path, owner: str) -> bool:
+ try:
+ return (lock_path / "owner").read_text(encoding="utf-8").strip() == owner
+ except OSError:
+ return False
+
+ def _acquire_lock_dir(self, lock_path: Path) -> str:
+ owner = f"{os.getpid()} {_PROCESS_TOKEN}.{uuid4().hex}"
+ owner_path = lock_path / "owner"
+ for _ in range(_LOCK_ATTEMPTS):
+ try:
+ os.mkdir(lock_path)
+ try:
+ os.chmod(lock_path, 0o700)
+ owner_path.write_text(f"{owner}\n", encoding="utf-8")
+ os.chmod(owner_path, 0o600)
+ except OSError:
+ lock_path.rmdir()
+ raise
+ return owner
+ except FileExistsError:
+ try:
+ stale = time.time() - lock_path.stat().st_mtime > _LOCK_STALE_AFTER
+ except FileNotFoundError:
+ continue
+ if stale:
+ try:
+ recorded = owner_path.read_text(encoding="utf-8")
+ except (OSError, UnicodeError):
+ recorded = None
+ if recorded is None or self._owner_dead(recorded):
+ with contextlib.suppress(FileNotFoundError):
+ owner_path.unlink()
+ with contextlib.suppress(FileNotFoundError):
+ lock_path.rmdir()
+ continue
+ time.sleep(_LOCK_RETRY_INTERVAL)
+ raise RuntimeError(f"harness state lock not acquired: {lock_path}")
+
+ @contextlib.contextmanager
+ def _locked(self) -> Iterator[None]:
+ if self.file_path is not None:
+ target = Path(os.path.realpath(self.file_path))
+ target.parent.mkdir(parents=True, exist_ok=True)
+ lock_path = target.with_name(f"{target.name}.lock")
+ self._held_lock = (lock_path, self._acquire_lock_dir(lock_path))
+ try:
+ self._sync_from_disk()
+ yield
+ finally:
+ if self._held_lock is not None:
+ lock_path, owner = self._held_lock
+ if self._owner_matches(lock_path, owner):
+ (lock_path / "owner").unlink()
+ lock_path.rmdir()
+ self._held_lock = None
+
def load(self) -> "HarnessState":
if self.file_path is None or not self.file_path.exists():
self._loaded_mtime = None
@@ -522,6 +611,8 @@ class HarnessState:
if self.file_path is None:
# in_memory fallback: nothing to persist.
return self
+ if self._held_lock is None:
+ raise RuntimeError("harness state lock not acquired")
self.file_path.parent.mkdir(parents=True, exist_ok=True)
data = {
"schema": 1,
@@ -544,9 +635,23 @@ class HarnessState:
descriptor = os.open(temp_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)
with os.fdopen(descriptor, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
+ f.flush()
+ if sys.platform == "darwin":
+ fcntl.fcntl(descriptor, fcntl.F_FULLFSYNC)
+ else:
+ os.fsync(descriptor)
if existing_mode is not None:
os.chmod(temp_path, existing_mode)
+ lock_path, owner = self._held_lock
+ if not self._owner_matches(lock_path, owner):
+ raise RuntimeError(f"harness state lock lost: {lock_path}")
os.replace(temp_path, target_path)
+ if os.name == "posix":
+ directory_fd = os.open(target_path.parent, os.O_RDONLY)
+ try:
+ os.fsync(directory_fd)
+ finally:
+ os.close(directory_fd)
finally:
temp_path.unlink(missing_ok=True)
self._loaded_mtime = self._disk_mtime()
@@ -581,18 +686,18 @@ class HarnessState:
source=source,
)
self._ensure_local_writable()
- self._sync_from_disk()
- return self._upsert(
- kind,
- title,
- content,
- id=id,
- path=path,
- reference=reference,
- arguments=arguments,
- metadata=metadata,
- source=source,
- )
+ with self._locked():
+ return self._upsert(
+ kind,
+ title,
+ content,
+ id=id,
+ path=path,
+ reference=reference,
+ arguments=arguments,
+ metadata=metadata,
+ source=source,
+ )
def _upsert(
self,
@@ -688,14 +793,14 @@ class HarnessState:
if target := self._global_target(global_, kwargs):
return target.delete(kind, id)
self._ensure_local_writable()
- self._sync_from_disk()
- if kind not in self.entries:
- raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")
- if id not in self.entries[kind]:
- return False
- del self.entries[kind][id]
- self.save()
- return True
+ with self._locked():
+ if kind not in self.entries:
+ raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")
+ if id not in self.entries[kind]:
+ return False
+ del self.entries[kind][id]
+ self.save()
+ return True
def list(self, kind: HarnessKind | None = None, *, global_: bool = False, **kwargs: Any) -> list[HarnessEntry]:
if target := self._global_target(global_, kwargs):
@@ -738,26 +843,26 @@ class HarnessState:
source=source,
)
self._ensure_local_writable()
- self._sync_from_disk()
- if kind not in self.entries:
- raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")
- _require_text(kind, _describe_entry(id, title), "title", title)
- if id is not None:
- _require_text(kind, _describe_entry(id, title), "id", id)
- entry_id = id or _slug(title, kind)
- if entry_id in self.entries[kind]:
- raise ValueError(f"{kind} entry {entry_id!r} already exists")
- return self._upsert(
- kind,
- title,
- content,
- id=entry_id,
- path=path,
- reference=reference,
- arguments=arguments,
- metadata=metadata,
- source=source,
- )
+ with self._locked():
+ if kind not in self.entries:
+ raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")
+ _require_text(kind, _describe_entry(id, title), "title", title)
+ if id is not None:
+ _require_text(kind, _describe_entry(id, title), "id", id)
+ entry_id = id or _slug(title, kind)
+ if entry_id in self.entries[kind]:
+ raise ValueError(f"{kind} entry {entry_id!r} already exists")
+ return self._upsert(
+ kind,
+ title,
+ content,
+ id=entry_id,
+ path=path,
+ reference=reference,
+ arguments=arguments,
+ metadata=metadata,
+ source=source,
+ )
def update(
self,
@@ -788,23 +893,23 @@ class HarnessState:
source=source,
)
self._ensure_local_writable()
- self._sync_from_disk()
- if kind not in self.entries:
- raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")
- _require_text(kind, _describe_entry(id, title), "id", id)
- if id not in self.entries[kind]:
- raise ValueError(f"{kind} entry {id!r} does not exist")
- return self._upsert(
- kind,
- title,
- content,
- id=id,
- path=path,
- reference=reference,
- arguments=arguments,
- metadata=metadata,
- source=source,
- )
+ with self._locked():
+ if kind not in self.entries:
+ raise ValueError(f"unknown harness kind {kind!r}; expected one of {_KINDS}")
+ _require_text(kind, _describe_entry(id, title), "id", id)
+ if id not in self.entries[kind]:
+ raise ValueError(f"{kind} entry {id!r} does not exist")
+ return self._upsert(
+ kind,
+ title,
+ content,
+ id=id,
+ path=path,
+ reference=reference,
+ arguments=arguments,
+ metadata=metadata,
+ source=source,
+ )
def create_memory(
self,
@@ -1047,24 +1152,24 @@ class HarnessState:
if target := self._global_target(global_, kwargs):
return target.record_refinement(trigger, changes, evidence=evidence, outcome=outcome, id=id)
self._ensure_local_writable()
- self._sync_from_disk()
- _validate_refinement_event(trigger, changes, evidence=evidence, outcome=outcome)
- if id is not None and (not isinstance(id, str) or not id):
- raise ValueError(
- f"refinement event rejected: id must be a non-empty string when provided, got {_type_name(id)}"
+ with self._locked():
+ _validate_refinement_event(trigger, changes, evidence=evidence, outcome=outcome)
+ if id is not None and (not isinstance(id, str) or not id):
+ raise ValueError(
+ f"refinement event rejected: id must be a non-empty string when provided, got {_type_name(id)}"
+ )
+ event_id = id or f"refine_{len(self.refinements) + 1:04d}"
+ normalized_changes = [changes] if isinstance(changes, str) else list(changes)
+ event = RefinementEvent(
+ id=event_id,
+ trigger=trigger,
+ changes=normalized_changes,
+ evidence=evidence,
+ outcome=outcome,
)
- event_id = id or f"refine_{len(self.refinements) + 1:04d}"
- normalized_changes = [changes] if isinstance(changes, str) else list(changes)
- event = RefinementEvent(
- id=event_id,
- trigger=trigger,
- changes=normalized_changes,
- evidence=evidence,
- outcome=outcome,
- )
- self.refinements.append(event)
- self.save()
- return event
+ self.refinements.append(event)
+ self.save()
+ return event
def plan_refinement(
self,
diff --git a/prime-agent-runtime/test/test_harness.py b/prime-agent-runtime/test/test_harness.py
index 40b05a9e84..2f010b5f29 100644
--- a/prime-agent-runtime/test/test_harness.py
+++ b/prime-agent-runtime/test/test_harness.py
@@ -2,11 +2,13 @@ from __future__ import annotations
import json
import os
+import stat
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
+from unittest.mock import patch
from rlm import harness as package_harness
from rlm import rlm as callable_rlm
@@ -523,6 +525,133 @@ class HarnessStateTest(unittest.TestCase):
titles = [entry.title for entry in HarnessState(real_path).entries["memory"].values()]
self.assertEqual(titles, ["Seed", "Through alias"])
+ def test_concurrent_subprocess_creates_all_land(self) -> None:
+ with tempfile.TemporaryDirectory() as temp_dir:
+ state_path = Path(temp_dir) / "harness" / "harness_state.json"
+ start = Path(temp_dir) / "start"
+ env = dict(os.environ)
+ env["PYTHONPATH"] = str(Path(__file__).resolve().parents[1] / "src")
+ workers = []
+ for worker in range(8):
+ code = (
+ "import time\n"
+ "from pathlib import Path\n"
+ "from rlm.harness import HarnessState\n"
+ f"while not Path({str(start)!r}).exists():\n"
+ " time.sleep(0.005)\n"
+ f"state = HarnessState({str(state_path)!r})\n"
+ "for i in range(10):\n"
+ f" state.create('memory', f'note {worker} {{i}}', f'note {worker} {{i}}', id=f'm-{worker}-{{i}}')\n"
+ )
+ workers.append(
+ subprocess.Popen([sys.executable, "-c", code], env=env)
+ )
+ start.write_text("", encoding="utf-8")
+ for worker in workers:
+ self.assertEqual(worker.wait(), 0)
+ entries = HarnessState(state_path).list("memory")
+ self.assertEqual(len(entries), 80)
+
+ def test_held_lock_dir_defers_writes_and_pins_the_shared_lock_path(self) -> None:
+ with tempfile.TemporaryDirectory() as temp_dir:
+ state_path = Path(temp_dir) / "harness" / "harness_state.json"
+ state = HarnessState(state_path)
+ state.create("memory", "Seed", "Creates the file.", id="seed")
+ lock_dir = Path(f"{os.path.realpath(state_path)}.lock")
+ lock_dir.mkdir()
+ try:
+ with self.assertRaises(RuntimeError) as raised:
+ state.create("memory", "Second", "Blocked by the lock.", id="second")
+ finally:
+ lock_dir.rmdir()
+ self.assertIn(str(lock_dir), str(raised.exception))
+ entries = HarnessState(state_path).list("memory")
+ self.assertEqual([entry.id for entry in entries], ["seed"])
+
+ def test_save_syncs_parent_after_replace(self) -> None:
+ if os.name != "posix":
+ self.skipTest("directory fsync is POSIX-only")
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = Path(temp_dir) / "harness_state.json"
+ state = HarnessState(path)
+ events: list[str] = []
+ replace, fsync = os.replace, os.fsync
+
+ def observed_replace(source: Path, destination: Path) -> None:
+ replace(source, destination)
+ events.append("replace")
+
+ def observed_fsync(fd: int) -> None:
+ fsync(fd)
+ events.append("directory sync" if stat.S_ISDIR(os.fstat(fd).st_mode) else "file sync")
+
+ with patch.object(os, "replace", side_effect=observed_replace), patch.object(
+ os, "fsync", side_effect=observed_fsync
+ ):
+ state.create_memory("Durable", "Saved")
+ self.assertEqual(events[-2:], ["replace", "directory sync"])
+
+ def test_live_stale_lock_is_not_stolen(self) -> None:
+ with tempfile.TemporaryDirectory() as temp_dir:
+ state_path = Path(temp_dir) / "harness_state.json"
+ state = HarnessState(state_path)
+ state.create_memory("Seed", "Present")
+ lock_dir = Path(f"{os.path.realpath(state_path)}.lock")
+ owner = state._acquire_lock_dir(lock_dir)
+ os.utime(lock_dir, (1, 1))
+ try:
+ with self.assertRaises(RuntimeError):
+ state.create_memory("Second", "Blocked")
+ self.assertTrue(state._owner_matches(lock_dir, owner))
+ finally:
+ (lock_dir / "owner").unlink()
+ lock_dir.rmdir()
+ self.assertEqual(len(HarnessState(state_path).list("memory")), 1)
+
+ def test_dead_lock_is_reclaimed_without_old_owner_cleanup(self) -> None:
+ with tempfile.TemporaryDirectory() as temp_dir:
+ state_path = Path(temp_dir) / "harness_state.json"
+ state = HarnessState(state_path)
+ state.create_memory("Seed", "Present")
+ lock_dir = Path(f"{os.path.realpath(state_path)}.lock")
+ owner = state._acquire_lock_dir(lock_dir)
+ worker = subprocess.Popen([sys.executable, "-c", "pass"])
+ worker.wait()
+ (lock_dir / "owner").write_text(f"{worker.pid} dead-token\n", encoding="utf-8")
+ os.utime(lock_dir, (1, 1))
+ state.create_memory("Second", "Saved")
+ self.assertFalse(state._owner_matches(lock_dir, owner))
+ self.assertFalse(lock_dir.exists())
+ self.assertEqual(len(HarnessState(state_path).list("memory")), 2)
+
+ def test_unparseable_stale_owner_is_reclaimed(self) -> None:
+ with tempfile.TemporaryDirectory() as temp_dir:
+ state_path = Path(temp_dir) / "harness_state.json"
+ state = HarnessState(state_path)
+ state.create_memory("Seed", "Present")
+ lock_dir = Path(f"{os.path.realpath(state_path)}.lock")
+ lock_dir.mkdir()
+ (lock_dir / "owner").write_bytes(b"\xff")
+ os.utime(lock_dir, (1, 1))
+ state.create_memory("Second", "Saved")
+ self.assertFalse(lock_dir.exists())
+ self.assertEqual(len(HarnessState(state_path).list("memory")), 2)
+
+ def test_save_refuses_lost_lock(self) -> None:
+ with tempfile.TemporaryDirectory() as temp_dir:
+ state_path = Path(temp_dir) / "harness_state.json"
+ state = HarnessState(state_path)
+ state.create_memory("Seed", "Present")
+ with self.assertRaisesRegex(RuntimeError, "lock lost"):
+ with state._locked():
+ lock_dir, _ = state._held_lock
+ (lock_dir / "owner").write_text("999999 other-token\n", encoding="utf-8")
+ state.save()
+ self.assertTrue(lock_dir.exists())
+ (lock_dir / "owner").unlink()
+ lock_dir.rmdir()
+ self.assertEqual(len(HarnessState(state_path).list("memory")), 1)
+
def test_load_ignores_unknown_json_keys(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
state_path = Path(temp_dir) / "harness_state.json"