The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
//! Cross-process exclusive locking for the auth-profile store: acquisition,
//! stale-lock reclamation, and same-process leaked-lock self-healing.
use anyhow::{Context, Result};
use std::fs::{self, OpenOptions};
use std::io::Write;
use std::sync::TryLockError;
use std::thread;
use std::time::{Duration, Instant};
#[cfg(test)]
use std::sync::atomic::Ordering;
use super::{
annotate_lock_create_failure, in_process_lock_for, is_pid_alive, AuthProfileLockGuard,
AuthProfilesStore, LOCK_TIMEOUT_MS, LOCK_WAIT_MS, MALFORMED_LOCK_GRACE_MS, STALE_LOCK_AGE_MS,
};
impl AuthProfilesStore {
pub(super) fn acquire_lock(&self) -> Result<AuthProfileLockGuard> {
// Test-only: simulate a full / read-only filesystem that can't create
// the lock file, to drive the read-only fallback in `load`.
#[cfg(test)]
if self.force_lock_unwritable.swap(false, Ordering::SeqCst) {
let io = std::io::Error::from(std::io::ErrorKind::StorageFull);
return Err(annotate_lock_create_failure(
anyhow::Error::new(io).context("open lock file"),
));
}
if let Some(parent) = self.lock_path.parent() {
fs::create_dir_all(parent)
.with_context(|| "Failed to create auth profile lock directory".to_string())?;
}
// Drive timeout + stale-recheck off wall-clock elapsed time, not the
// sum of explicit `thread::sleep(LOCK_WAIT_MS)` calls. The earlier
// counter-based approach excluded time spent inside
// `retry_with_backoff` (which can sleep up to ~30s on its own
// schedule before returning AlreadyExists) and the lock-file I/O
// syscalls. Under Windows AV contention that drift could push
// both `LOCK_TIMEOUT_MS` and `next_stale_recheck_ms` significantly
// later than intended.
let started_at = Instant::now();
// Serialize same-process acquirers on an in-memory lock keyed by this
// path before we ever touch the on-disk lock file. Two things depend on
// holding it: (1) concurrent `app_state_snapshot` calls in this process
// queue here instead of racing `create_new`/`Drop` on the file, and
// (2) it lets us treat an on-disk lock recording our own pid as a leaked
// `Drop` unlink and reclaim it immediately — no other thread in this
// process can hold a live guard while we own this in-memory lock (see
// `reclaim_self_owned_lock`). Held for the lifetime of the returned
// guard.
//
// Use `try_lock` against the *same* `LOCK_TIMEOUT_MS` budget as the
// on-disk wait rather than a blocking `lock()`: a wedged in-process
// holder must not be able to strand an RPC/blocking worker past the
// timeout the caller (e.g. `app_state_snapshot`) expects. Poison is
// recoverable — the `()` payload carries no invariant.
let in_process_lock = in_process_lock_for(&self.lock_path);
let in_process_guard = loop {
match in_process_lock.try_lock() {
Ok(guard) => break guard,
Err(TryLockError::Poisoned(poisoned)) => break poisoned.into_inner(),
Err(TryLockError::WouldBlock) => {
if started_at.elapsed().as_millis() as u64 >= LOCK_TIMEOUT_MS {
anyhow::bail!("Timed out waiting for auth profile lock");
}
thread::sleep(Duration::from_millis(LOCK_WAIT_MS));
}
}
};
let mut cleared_stale = false;
// Periodically re-probe for stale locks during the busy-wait. A
// lock that started fresh (live pid, recent mtime) can age past
// STALE_LOCK_AGE_MS while we wait, and we want to recover from
// that without bailing at the LOCK_TIMEOUT_MS boundary.
let mut next_stale_recheck_ms: u64 = 1_000;
loop {
let open_result =
crate::util::retry_with_backoff("create auth profile lock", 6, 100, || {
OpenOptions::new()
.create_new(true)
.write(true)
.open(&self.lock_path)
.context("open lock file")
});
match open_result {
Ok(mut file) => {
// Issue #1612 — writing the pid line is what later lets
// a future acquirer recognise a crashed owner; if the
// write fails we must NOT report the lock as held with
// a malformed/empty file behind us, or stale recovery
// would silently degrade to the full 10s timeout for
// every subsequent acquire.
if let Err(e) = writeln!(file, "pid={}", std::process::id()) {
let _ = fs::remove_file(&self.lock_path);
return Err(e).with_context(|| {
"Failed to write auth profile lock owner".to_string()
});
}
return Ok(AuthProfileLockGuard {
lock_path: self.lock_path.clone(),
_in_process: in_process_guard,
});
}
Err(e) => {
let is_already_exists = e
.chain()
.find_map(|e| e.downcast_ref::<std::io::Error>())
.is_some_and(|ioe| ioe.kind() == std::io::ErrorKind::AlreadyExists);
if is_already_exists {
// A lock file recording our own pid can only be a
// leaked `Drop` unlink: we hold the in-process lock, so
// no live same-process guard exists. Reclaim it
// immediately rather than spinning the 30s age floor
// (`STALE_LOCK_AGE_MS`) that sits *behind* the ~10s RPC
// timeout — that gap is what produced the sustained
// "Timed out waiting for auth profile lock" retry storm
// (Sentry TAURI-RUST-B1 / #2318). Cheap enough (one tiny
// read) to re-probe every spin, which also self-heals a
// lock our own `Drop` leaks mid-wait.
if self.reclaim_self_owned_lock() {
continue;
}
// Issue #1612 — a previous openhuman crash can leave a
// stale auth-profiles.lock behind (a *different*, now-dead
// pid, or an aged leak), after which every RPC path that
// touches the auth profile store fails for the
// `LOCK_TIMEOUT_MS` window and the user gets stuck in a
// retry storm. Before falling back to the busy-wait, try
// once to peek at the writer's recorded PID and remove
// the lock if that process is no longer alive. Flag is
// flipped on the first probe (not only on success) so a
// live-pid / malformed / unreadable lock doesn't trigger
// a fresh sysinfo probe + log line on every busy-wait
// iteration.
if !cleared_stale {
cleared_stale = true;
if self.clear_lock_if_stale() {
continue;
}
} else {
let elapsed_ms = started_at.elapsed().as_millis() as u64;
if elapsed_ms >= next_stale_recheck_ms {
// The age-based reclaim check is cheap (one
// `fs::metadata` call in the common case) and
// safely no-ops on fresh, legitimate locks.
// Re-probing periodically lets us recover from
// a leaked-mid-wait lock without bailing at
// the 10s timeout.
next_stale_recheck_ms = next_stale_recheck_ms.saturating_add(1_000);
if self.clear_lock_if_stale() {
continue;
}
}
}
if started_at.elapsed().as_millis() as u64 >= LOCK_TIMEOUT_MS {
anyhow::bail!("Timed out waiting for auth profile lock");
}
thread::sleep(Duration::from_millis(LOCK_WAIT_MS));
} else {
// Sentry OPENHUMAN-TAURI-H8 collapses every
// non-AlreadyExists, non-transient `create_new`
// failure into a single fingerprint with no
// breadcrumb of which OS code actually fired.
// `annotate_lock_create_failure` embeds the
// underlying `io::ErrorKind` + `raw_os_error()` so
// future events split by root cause and we can
// widen `is_transient_fs_error` (or fix the
// underlying condition) for whichever code is hot.
return Err(annotate_lock_create_failure(e));
}
}
}
}
}
/// Returns `true` if an existing lock file was detected as stale and
/// successfully removed. Two cases reclaim:
///
/// 1. The recorded `pid=` line points at a process that is no longer
/// running — classic crashed-owner recovery (Issue #1612).
/// 2. The lock file's mtime is older than [`STALE_LOCK_AGE_MS`]. This
/// catches a *different* still-alive process that leaked its lock (its
/// `AuthProfileLockGuard::drop` could not unlink the file — e.g. Windows
/// AV / indexer briefly held a handle — and orphaned it with that live
/// pid inside). No legitimate auth-profile op holds the lock long enough
/// to be affected, so a too-old lock is unambiguously a leak. A lock
/// leaked by *this* process is handled far sooner — immediately, without
/// the age floor — by [`reclaim_self_owned_lock`](Self::reclaim_self_owned_lock),
/// which is sound because acquirers serialize on the in-process lock.
///
/// 3. The lock file has no parseable `pid=` line and is older than
/// [`MALFORMED_LOCK_GRACE_MS`]. A healthy holder writes its pid within
/// microseconds of `create_new`, so a pidless lock past that short
/// grace is an abandoned in-flight writer (crashed/killed between
/// `create_new` and the `pid=` write) — reclaim it rather than make
/// every reader spin the full [`STALE_LOCK_AGE_MS`]/`LOCK_TIMEOUT_MS`
/// window (the ~30s "stuck on Initializing OpenHuman" after a
/// kill+reopen). The grace is short but non-zero so we never reclaim a
/// live writer that is mid-`create_new`/`pid=`.
pub(super) fn clear_lock_if_stale(&self) -> bool {
let metadata = match fs::metadata(&self.lock_path) {
Ok(m) => m,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return false,
Err(e) => {
tracing::warn!(
target: "auth-profiles",
"[credentials] failed to stat lock file at {} for stale check: {e}",
self.lock_path.display()
);
return false;
}
};
let age = metadata
.modified()
.ok()
.and_then(|mtime| std::time::SystemTime::now().duration_since(mtime).ok());
let too_old = age.is_some_and(|a| a >= Duration::from_millis(STALE_LOCK_AGE_MS));
// A pidless lock needs only a short grace: no healthy holder leaves the
// file without a `pid=` line for more than the microsecond gap between
// `create_new` and the write, so anything older is abandoned. If mtime
// is unreadable (clock skew, platform limitation) default to stale —
// no legitimate in-flight writer would be undetectable for that long.
let malformed_too_old =
age.is_none_or(|a| a >= Duration::from_millis(MALFORMED_LOCK_GRACE_MS));
let content = match fs::read_to_string(&self.lock_path) {
Ok(s) => s,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return false,
Err(e) => {
tracing::warn!(
target: "auth-profiles",
"[credentials] failed to read lock file at {} for stale check: {e}",
self.lock_path.display()
);
return false;
}
};
let pid = content
.lines()
.find_map(|line| line.trim().strip_prefix("pid=")?.trim().parse::<u32>().ok());
let reclaim_reason: Option<String> = match pid {
Some(pid) if !is_pid_alive(pid) => Some(format!("pid {pid} not alive")),
Some(pid) if too_old => Some(format!(
"lock file older than {STALE_LOCK_AGE_MS}ms (recorded pid {pid}, presumed leaked)"
)),
None if malformed_too_old => Some(format!(
"no parseable pid and older than {MALFORMED_LOCK_GRACE_MS}ms \
(abandoned in-flight lock, reclaiming)"
)),
Some(_) => return false,
None => {
tracing::warn!(
target: "auth-profiles",
"[credentials] lock at {} has no parseable pid line and is younger than \
{MALFORMED_LOCK_GRACE_MS}ms; leaving in place briefly",
self.lock_path.display()
);
return false;
}
};
let Some(reason) = reclaim_reason else {
return false;
};
match fs::remove_file(&self.lock_path) {
Ok(()) => {
tracing::info!(
target: "auth-profiles",
"[credentials] removed stale auth profile lock at {} ({reason})",
self.lock_path.display()
);
true
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => true,
Err(e) => {
tracing::warn!(
target: "auth-profiles",
"[credentials] failed to remove stale lock at {} ({reason}): {e}",
self.lock_path.display()
);
false
}
}
}
/// Remove the on-disk lock file **iff** it records this process's own pid.
/// Returns `true` when a self-owned lock was found and removed.
///
/// MUST only be called while holding this path's in-process lock (acquired
/// at the top of [`acquire_lock`](Self::acquire_lock) via
/// [`in_process_lock_for`]). That invariant is what makes the removal safe:
/// same-process acquirers serialize on the in-process lock, so no other
/// thread in this process can be holding a live `AuthProfileLockGuard` while
/// we run. A lock file carrying our own pid is therefore necessarily a leak
/// — a previous guard's `Drop` could not unlink it (on Windows, AV / the
/// Search indexer briefly hold a handle on the just-written file) and
/// orphaned it with our still-alive pid inside.
///
/// This is the fast self-heal for Sentry TAURI-RUST-B1 (#2318): without it
/// the only recovery for a leaked-but-live-pid lock is the age branch in
/// [`clear_lock_if_stale`](Self::clear_lock_if_stale), whose
/// [`STALE_LOCK_AGE_MS`] floor (30s) is far longer than the ~10s RPC
/// timeout, so every `app_state_snapshot` timed out and retry-stormed for
/// the whole 30s window before age-reclaim kicked in.
pub(crate) fn reclaim_self_owned_lock(&self) -> bool {
let me = std::process::id();
tracing::trace!(
target: "auth-profiles",
"[credentials] probing for self-owned leaked lock at {} (our pid {me})",
self.lock_path.display()
);
let content = match fs::read_to_string(&self.lock_path) {
Ok(s) => s,
// NotFound: already gone (raced with another reclaim / the guard's
// own Drop). Any other read error: fall back to the age-based and
// busy-wait paths rather than guess at ownership.
Err(e) => {
tracing::debug!(
target: "auth-profiles",
"[credentials] self-owned probe could not read lock at {} ({e}); \
deferring to stale/busy-wait recovery",
self.lock_path.display()
);
return false;
}
};
let pid = content
.lines()
.find_map(|line| line.trim().strip_prefix("pid=")?.trim().parse::<u32>().ok());
if pid != Some(me) {
tracing::trace!(
target: "auth-profiles",
"[credentials] lock at {} is not self-owned (recorded {pid:?}, our pid {me}); \
deferring to stale/busy-wait recovery",
self.lock_path.display()
);
return false;
}
match fs::remove_file(&self.lock_path) {
Ok(()) => {
tracing::info!(
target: "auth-profiles",
"[credentials] reclaimed leaked self-owned auth profile lock at {} \
(recorded our own live pid {me}; a prior guard's Drop unlink was blocked, \
most likely by Windows AV/indexer)",
self.lock_path.display()
);
true
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => true,
Err(e) => {
tracing::warn!(
target: "auth-profiles",
"[credentials] failed to reclaim self-owned auth profile lock at {}: {e}",
self.lock_path.display()
);
false
}
}
}
}