Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

fix(storage): log lease-lock retries; the probe test waits for real contention

M3gA-Mind2bc2246d046e
diff --git a/crates/openhuman-core/src/storage/lease_local.rs b/crates/openhuman-core/src/storage/lease_local.rs
index 10b6550024..5c7ff95b2a 100644
--- a/crates/openhuman-core/src/storage/lease_local.rs
+++ b/crates/openhuman-core/src/storage/lease_local.rs
@@ -73,16 +73,37 @@ fn is_contended(error: &io::Error) -> bool {
/// A blocking sleep: the caller holds the `held` std mutex, so it can't await.
// ponytail: a bounded retry absorbs probes; a probe that takes no lock
// (F_GETLK) would remove the race outright.
-fn lock_past_probes(file: &File) -> io::Result<()> {
- for _ in 1..PROBE_ATTEMPTS {
+fn lock_past_probes(file: &File, key: &str, node: &str) -> io::Result<()> {
+ let mut retries = 0;
+ let result = loop {
match file.try_lock_exclusive() {
- Err(error) if is_contended(&error) => std::thread::sleep(PROBE_WAIT),
- other => return other,
+ Err(error) if is_contended(&error) && retries + 1 < PROBE_ATTEMPTS => {
+ #[cfg(test)]
+ CONTENDED_ATTEMPTS.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ retries += 1;
+ std::thread::sleep(PROBE_WAIT);
+ }
+ other => break other,
}
+ };
+ if retries > 0 {
+ tracing::debug!(
+ target: "openhuman::storage::lease",
+ key,
+ node,
+ retries,
+ locked = result.is_ok(),
+ "[lease] local acquire retried a contended lock"
+ );
}
- file.try_lock_exclusive()
+ result
}
+/// Contended attempts `lock_past_probes` has retried, so a test can release
+/// its stand-in probe lock only once `acquire` has actually met it.
+#[cfg(test)]
+static CONTENDED_ATTEMPTS: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
+
impl LocalLeases {
/// Leases under `root` for `node`.
pub fn new(root: impl Into<PathBuf>, node: impl Into<String>) -> Self {
@@ -188,7 +209,7 @@ impl LeaseStore for LocalLeases {
});
}
let file = self.open_lock(key)?;
- if let Err(error) = lock_past_probes(&file) {
+ if let Err(error) = lock_past_probes(&file, key, &self.node) {
if !is_contended(&error) {
let path = local_dir(&self.root, key).join(LOCK_FILE);
return Err(io_error("lock", &path, &error).into());
diff --git a/crates/openhuman-core/src/storage/lease_local_tests.rs b/crates/openhuman-core/src/storage/lease_local_tests.rs
index c8bb69c01a..8c1ddc7c5b 100644
--- a/crates/openhuman-core/src/storage/lease_local_tests.rs
+++ b/crates/openhuman-core/src/storage/lease_local_tests.rs
@@ -187,8 +187,16 @@ async fn acquire_rides_out_a_transient_inspection_lock() {
.open(key_dir.join(LOCK_FILE))
.unwrap();
probe.try_lock_exclusive().unwrap();
+ // Release the stand-in probe only after `acquire` has met it contended,
+ // so the test always exercises the retry rather than racing a timer.
+ let seen = CONTENDED_ATTEMPTS.load(std::sync::atomic::Ordering::SeqCst);
let release = std::thread::spawn(move || {
- std::thread::sleep(std::time::Duration::from_millis(2));
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ while CONTENDED_ATTEMPTS.load(std::sync::atomic::Ordering::SeqCst) == seen
+ && std::time::Instant::now() < deadline
+ {
+ std::thread::yield_now();
+ }
FileExt::unlock(&probe).unwrap();
});
@@ -196,4 +204,8 @@ async fn acquire_rides_out_a_transient_inspection_lock() {
release.join().unwrap();
let grant = grant.expect("a probe-length lock must not refuse the acquire");
assert_eq!((grant.epoch, grant.previous_unclean), (2, true));
+ assert!(
+ CONTENDED_ATTEMPTS.load(std::sync::atomic::Ordering::SeqCst) > seen,
+ "the acquire never met the probe lock"
+ );
}