Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

fix(storage): local lease acquire rides out inspection probes; durable record write

M3gA-Mind045d71816154
diff --git a/crates/openhuman-core/src/storage/lease_local.rs b/crates/openhuman-core/src/storage/lease_local.rs
index 0ae5ac188d..10b6550024 100644
--- a/crates/openhuman-core/src/storage/lease_local.rs
+++ b/crates/openhuman-core/src/storage/lease_local.rs
@@ -4,8 +4,8 @@
//! open for as long as it holds the key; the OS drops it when the process
//! dies, so liveness needs no clock and these leases never expire
//! (`expires_at_ms == u64::MAX`). The record itself sits beside the lock in
-//! `.lease.json`, written only by the lock holder (temp file + rename), so
-//! others can name the owner.
+//! `.lease.json`, written only by the lock holder (fsynced temp file +
+//! durable rename), so others can name the owner.
//!
//! A free lock over a record that is not released means the previous holder
//! died holding it: the next grant is `previous_unclean`.
@@ -30,7 +30,11 @@ use super::StorageError;
const LOCK_FILE: &str = ".lease";
const RECORD_FILE: &str = ".lease.json";
-const RECORD_TMP: &str = ".lease.json.tmp";
+/// Inspections (`holder`, the doctor's `holder_readonly`) test liveness by
+/// taking the lock for an instant, so `acquire` retries a contended lock
+/// for up to `PROBE_ATTEMPTS × PROBE_WAIT` before calling the key held.
+const PROBE_ATTEMPTS: u32 = 10;
+const PROBE_WAIT: std::time::Duration = std::time::Duration::from_millis(5);
struct Holding {
/// Kept open: closing it drops the lock.
@@ -65,6 +69,20 @@ fn is_contended(error: &io::Error) -> bool {
|| error.raw_os_error() == fs2::lock_contended_error().raw_os_error()
}
+/// `try_lock_exclusive`, riding out a lock held only by an inspection probe.
+/// A blocking sleep: the caller holds the `held` std mutex, so it can't await.
+// ponytail: a bounded retry absorbs probes; a probe that takes no lock
+// (F_GETLK) would remove the race outright.
+fn lock_past_probes(file: &File) -> io::Result<()> {
+ for _ in 1..PROBE_ATTEMPTS {
+ match file.try_lock_exclusive() {
+ Err(error) if is_contended(&error) => std::thread::sleep(PROBE_WAIT),
+ other => return other,
+ }
+ }
+ file.try_lock_exclusive()
+}
+
impl LocalLeases {
/// Leases under `root` for `node`.
pub fn new(root: impl Into<PathBuf>, node: impl Into<String>) -> Self {
@@ -119,13 +137,14 @@ impl LocalLeases {
}
fn write_record(&self, key: &str, record: &LeaseRecord) -> Result<(), StorageError> {
- let dir = local_dir(&self.root, key);
- let tmp = dir.join(RECORD_TMP);
+ let path = local_dir(&self.root, key).join(RECORD_FILE);
let bytes = serde_json::to_vec(record)
.map_err(|error| StorageError::serialization(error.to_string()))?;
- fs::write(&tmp, bytes).map_err(|error| io_error("write", &tmp, &error))?;
- let path = dir.join(RECORD_FILE);
- fs::rename(&tmp, &path).map_err(|error| io_error("rename", &path, &error))
+ // fsync + durable rename: a crash must not leave an empty record,
+ // which would read as malformed and block the key.
+ crate::security::secret_storage::file_store::write_atomic(&path, &bytes).map_err(|error| {
+ StorageError::backend(format!("lease write {}: {error}", path.display()))
+ })
}
/// Checks that `grant` is the one this instance holds and applies
@@ -169,7 +188,7 @@ impl LeaseStore for LocalLeases {
});
}
let file = self.open_lock(key)?;
- if let Err(error) = file.try_lock_exclusive() {
+ if let Err(error) = lock_past_probes(&file) {
if !is_contended(&error) {
let path = local_dir(&self.root, key).join(LOCK_FILE);
return Err(io_error("lock", &path, &error).into());
diff --git a/crates/openhuman-core/src/storage/lease_local_tests.rs b/crates/openhuman-core/src/storage/lease_local_tests.rs
index 7d026226ab..c8bb69c01a 100644
--- a/crates/openhuman-core/src/storage/lease_local_tests.rs
+++ b/crates/openhuman-core/src/storage/lease_local_tests.rs
@@ -158,3 +158,42 @@ async fn a_failed_release_write_keeps_the_lock_and_the_holding() {
}
a.release(grant).await.unwrap();
}
+
+#[tokio::test]
+async fn acquire_rides_out_a_transient_inspection_lock() {
+ // The doctor's `holder_readonly` (often another process) checks liveness
+ // by taking the lock for an instant; an `acquire` landing in that window
+ // must not report the dead previous owner as the holder.
+ let dir = tempfile::tempdir().unwrap();
+ let key_dir = local_dir(dir.path(), "k");
+ fs::create_dir_all(&key_dir).unwrap();
+ fs::write(
+ key_dir.join(RECORD_FILE),
+ serde_json::to_vec(&LeaseRecord {
+ owner: "dead-node".into(),
+ endpoint: None,
+ epoch: 1,
+ expires_at_ms: u64::MAX,
+ released: false,
+ })
+ .unwrap(),
+ )
+ .unwrap();
+ let probe = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(false)
+ .open(key_dir.join(LOCK_FILE))
+ .unwrap();
+ probe.try_lock_exclusive().unwrap();
+ let release = std::thread::spawn(move || {
+ std::thread::sleep(std::time::Duration::from_millis(2));
+ FileExt::unlock(&probe).unwrap();
+ });
+
+ let grant = LocalLeases::new(dir.path(), "b").acquire("k", 0).await;
+ release.join().unwrap();
+ let grant = grant.expect("a probe-length lock must not refuse the acquire");
+ assert_eq!((grant.epoch, grant.previous_unclean), (2, true));
+}