Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

heal torn worker-journal tails on open and on read (#3387)

Sebastian Müller1c6498e4d5f8
diff --git a/.changes/fix-journal-torn-tail.md b/.changes/fix-journal-torn-tail.md
new file mode 100644
index 0000000000..a0700be88b
--- /dev/null
+++ b/.changes/fix-journal-torn-tail.md
@@ -0,0 +1 @@
+- Daemon recovery journals now preserve intact records and safely remove an incomplete trailing record after a crash, including a cut in Unicode text, so subsequent records survive reopening. Healing syncs worker recovery data before replacement and stops on journal read failures instead of rewriting unreadable data.
diff --git a/crates/pa-core/src/platform/lock_dir.rs b/crates/pa-core/src/platform/lock_dir.rs
index c37ad39700..8decab2829 100644
--- a/crates/pa-core/src/platform/lock_dir.rs
+++ b/crates/pa-core/src/platform/lock_dir.rs
@@ -510,6 +510,17 @@ pub fn try_lock_exclusive(file: &std::fs::File) -> io::Result<()> {
file.try_lock().map_err(io::Error::other)
}
+/// Release an OS file lock without waiting for duplicated or inherited handles
+/// to close. The platform wall owns the Unix and Windows unlock semantics.
+///
+/// # Errors
+///
+/// Returns the underlying OS error when unlocking fails. Unlocking a handle
+/// without a held lock may succeed or return an error, depending on the platform.
+pub fn unlock_file(file: &std::fs::File) -> io::Result<()> {
+ file.unlock()
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/pa-core/src/platform/mod.rs b/crates/pa-core/src/platform/mod.rs
index 1b0f6bda9c..df19b698b7 100644
--- a/crates/pa-core/src/platform/mod.rs
+++ b/crates/pa-core/src/platform/mod.rs
@@ -10,7 +10,7 @@ pub mod process;
pub mod shell;
pub use fs::fsync;
-pub use lock_dir::{lock_exclusive, try_lock_exclusive, LockDir};
+pub use lock_dir::{lock_exclusive, try_lock_exclusive, unlock_file, LockDir};
pub use perms::{
file_mode, is_executable, is_readable_writable, restrict_dir, restrict_file, set_private_mode,
};
diff --git a/crates/pa-core/tests/fixtures/protocol_caps_kernel.py b/crates/pa-core/tests/fixtures/protocol_caps_kernel.py
new file mode 100755
index 0000000000..d497c26c43
--- /dev/null
+++ b/crates/pa-core/tests/fixtures/protocol_caps_kernel.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+import json
+import sys
+import time
+
+print(json.dumps({"event": "ready", "protocol": 3, "python": "3.13.0"}), flush=True)
+for line in sys.stdin:
+ try:
+ req = json.loads(line)
+ except Exception:
+ continue
+ if req.get("code") == "corrupt-huge-line":
+ sys.stdout.write("x" * (33 * 1024 * 1024))
+ sys.stdout.flush()
+ time.sleep(30)
+ break
+ if req.get("code") in ("big-frame", "unicode-blank-frame"):
+ padding = " " * (31 * 1024 * 1024) if req.get("code") == "big-frame" else " \t\r\v\f\u0085\u00a0\u2000\u2028\u2029\u3000 "
+ sys.stdout.write(padding + "\n")
+ sys.stdout.write(json.dumps({"event": "stdout", "id": req.get("id"), "text": "big"}) + "\n")
+ sys.stdout.write(json.dumps({"event": "done", "id": req.get("id"), "status": "ok"}) + "\n")
+ sys.stdout.flush()
diff --git a/crates/pa-core/tests/kernel_protocol_frame_caps.rs b/crates/pa-core/tests/kernel_protocol_frame_caps.rs
index 84dcc94eab..53781a95c4 100644
--- a/crates/pa-core/tests/kernel_protocol_frame_caps.rs
+++ b/crates/pa-core/tests/kernel_protocol_frame_caps.rs
@@ -16,7 +16,6 @@
#![cfg(unix)]
use std::collections::HashMap;
-use std::os::unix::fs::PermissionsExt;
use std::time::{Duration, Instant};
use pa_core::kernel::manager::{KernelStartOptions, ReplKernelManager};
@@ -27,38 +26,10 @@ use pa_core::kernel::shared::{
/// Speaks protocol v3: ready, then per request the oversized unterminated
/// line (the corruption under test), a multi-MiB blank line followed by a
/// plain stdout frame (the linear-scan test), or a plain done frame.
-const FAKE_RUNTIME: &str = r#"#!/usr/bin/env python3
-import json
-import sys
-import time
-
-print(json.dumps({"event": "ready", "protocol": 3, "python": "3.13.0"}), flush=True)
-for line in sys.stdin:
- try:
- req = json.loads(line)
- except Exception:
- continue
- if req.get("code") == "corrupt-huge-line":
- sys.stdout.write("x" * (33 * 1024 * 1024))
- sys.stdout.flush()
- time.sleep(30)
- break
- if req.get("code") in ("big-frame", "unicode-blank-frame"):
- padding = " " * (31 * 1024 * 1024) if req.get("code") == "big-frame" else " \t\r\v\f\u0085\u00a0\u2000\u2028\u2029\u3000 "
- sys.stdout.write(padding + "\n")
- sys.stdout.write(json.dumps({"event": "stdout", "id": req.get("id"), "text": "big"}) + "\n")
- sys.stdout.write(json.dumps({"event": "done", "id": req.get("id"), "status": "ok"}) + "\n")
- sys.stdout.flush()
-"#;
-
fn fake_runtime_path() -> std::path::PathBuf {
- let dir = tempfile::TempDir::new().expect("temp dir");
- let path = dir.path().join("fake-kernel");
- std::fs::write(&path, FAKE_RUNTIME).expect("write fake runtime");
- std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).expect("chmod");
- // Leak the temp dir: the script must exist until the spawned child exits.
- std::mem::forget(dir);
- path
+ // An immutable program avoids writable executable descriptors during concurrent spawns.
+ std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+ .join("tests/fixtures/protocol_caps_kernel.py")
}
fn manager(python: std::path::PathBuf) -> ReplKernelManager {
diff --git a/crates/pa-daemon/src/compaction_supervision.rs b/crates/pa-daemon/src/compaction_supervision.rs
index 42a7d4291f..707ef7cb0a 100644
--- a/crates/pa-daemon/src/compaction_supervision.rs
+++ b/crates/pa-daemon/src/compaction_supervision.rs
@@ -8,7 +8,7 @@ use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::Duration;
-use anyhow::Result;
+use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use serde_json::Map;
@@ -181,6 +181,7 @@ pub(crate) struct TerminalCompactionRecord {
/// worker descriptors, latest record per session. Declared records feed
/// the replacement-worker create replay until they are consumed.
pub(crate) struct TerminalCompactionJournal {
+ _write_owner: crate::journal::JournalWriteOwner,
path: PathBuf,
latest: HashMap<String, TerminalCompactionRecord>,
/// Failed durable writes stay retryable in memory (the replay retries them).
@@ -194,19 +195,35 @@ impl TerminalCompactionJournal {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
+ let write_owner = crate::journal::journal_write_owner(path)?;
+ let latest = Self::load(path)?;
+ if crate::journal::tail_is_torn(path)? {
+ let records = latest
+ .values()
+ .map(serde_json::to_value)
+ .collect::<std::result::Result<Vec<_>, _>>()?;
+ crate::journal::rewrite_records(path, &records, crate::journal::Finalize::Synced)?;
+ }
Ok(TerminalCompactionJournal {
- latest: Self::load(path),
+ _write_owner: write_owner,
+ latest,
retryable: HashMap::new(),
path: path.to_path_buf(),
})
}
- fn load(path: &Path) -> HashMap<String, TerminalCompactionRecord> {
- let Ok(content) = std::fs::read_to_string(path) else {
- return HashMap::new();
+ fn load(path: &Path) -> Result<HashMap<String, TerminalCompactionRecord>> {
+ let bytes = match std::fs::read(path) {
+ Ok(bytes) => bytes,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
+ return Ok(HashMap::new());
+ }
+ Err(error) => {
+ return Err(error).with_context(|| format!("read journal {}", path.display()));
+ }
};
let mut latest = HashMap::new();
- for line in content.lines() {
+ for line in crate::journal::decode_journal_lines(&bytes, path)? {
if line.trim().is_empty() {
continue;
}
@@ -215,7 +232,7 @@ impl TerminalCompactionJournal {
};
latest.insert(record.active_session_id.clone(), record);
}
- latest
+ Ok(latest)
}
/// Record the terminal declaration (durable before the synthetic end goes out, so a
@@ -291,6 +308,20 @@ pub(crate) fn abort_grace() -> Duration {
}
impl crate::supervisor::Supervisor {
+ pub(crate) fn with_compaction_journal<T>(
+ &self,
+ action: impl FnOnce(&mut TerminalCompactionJournal) -> Result<T>,
+ ) -> Result<T> {
+ let mut journal = self
+ .compaction_journal
+ .lock()
+ .expect("compaction journal lock");
+ let journal = journal
+ .as_mut()
+ .context("terminal compaction journal is not initialized")?;
+ action(journal)
+ }
+
/// The `abort_compaction` supervisor arm: acknowledge immediately (TS `abortCompaction`
/// is always instant), forward best-effort, declare terminal a run past the grace.
pub(crate) async fn handle_abort_compaction(
@@ -447,13 +478,9 @@ impl crate::supervisor::Supervisor {
) {
let session_file = resident.descriptor.lock().await.session_file.clone();
let declared = crate::util::now_iso();
- let terminal = {
- let mut journal = self
- .compaction_journal
- .lock()
- .expect("compaction journal lock");
+ let terminal = self.with_compaction_journal(|journal| {
let Some(terminal) = take() else {
- return;
+ return Ok(None);
};
let record = TerminalCompactionRecord {
version: 1,
@@ -469,7 +496,17 @@ impl crate::supervisor::Supervisor {
terminal.active_session_id
));
}
- terminal
+ Ok(Some(terminal))
+ });
+ let terminal = match terminal {
+ Ok(Some(terminal)) => terminal,
+ Ok(None) => return,
+ Err(error) => {
+ self.log_line(&format!(
+ "terminal compaction journal unavailable: {error:#}"
+ ));
+ return;
+ }
};
self.note_compaction_abort_declared();
// The synthetic `compaction_end`: an abort carries `aborted: true` with no error
@@ -710,6 +747,7 @@ mod tests {
})
.unwrap();
journal.clear("session-b").unwrap();
+ drop(journal);
let mut reopened = TerminalCompactionJournal::open(&path).unwrap();
assert_eq!(reopened.pending("session-b").unwrap(), None);
}
@@ -796,6 +834,7 @@ mod tests {
std::fs::remove_dir(&path).unwrap();
let pending = failing.pending("session-a").unwrap().expect("retried");
assert_eq!(pending.reason, "threshold");
+ drop(failing);
let reloaded = TerminalCompactionJournal::open(&path).unwrap();
assert!(reloaded.latest.contains_key("session-a"));
@@ -812,4 +851,194 @@ mod tests {
);
let _ = std::fs::remove_dir_all(&dir);
}
+
+ #[test]
+ fn terminal_journal_read_failure_stops_recovery() {
+ let dir = std::env::temp_dir().join(format!("pa-comp-sup-{}", uuid::Uuid::new_v4()));
+ let path = dir.join("unreadable.jsonl");
+ std::fs::create_dir_all(&path).unwrap();
+ let sentinel = path.join("preserve");
+ std::fs::write(&sentinel, b"original data").unwrap();
+
+ assert!(TerminalCompactionJournal::open(&path).is_err());
+ assert_eq!(std::fs::read(&sentinel).unwrap(), b"original data");
+ std::fs::remove_dir_all(&dir).unwrap();
+ }
+
+ #[test]
+ fn terminal_journal_open_heals_torn_utf8_tail() {
+ let dir = std::env::temp_dir().join(format!("pa-comp-sup-{}", uuid::Uuid::new_v4()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join("compaction-supervision.jsonl");
+ let mut torn: Vec<u8> = br#"{"version":1,"type":"terminal_compaction","activeSessionId":"session-a","sessionFile":"/sessions/a.jsonl","reason":"threshold","declaredAt":"2026-09-23T00:00:00Z"}"#.to_vec();
+ torn.push(b'\n');
+ let whole = r#"{"activeSessionId":"session-b","reason":"shütdown"}"#;
+ torn.extend_from_slice(&whole.as_bytes()[..=whole.find('ü').unwrap()]);
+ assert!(std::str::from_utf8(&torn).is_err());
+ std::fs::write(&path, torn).unwrap();
+
+ {
+ let mut journal = TerminalCompactionJournal::open(&path).unwrap();
+ assert!(journal.pending("session-a").unwrap().is_some());
+ journal
+ .declare(TerminalCompactionRecord {
+ version: 1,
+ r#type: TERMINAL_COMPACTION_RECORD_TYPE.to_string(),
+ active_session_id: "session-b".to_string(),
+ session_file: None,
+ reason: "threshold".to_string(),
+ declared_at: "2026-09-23T00:00:01Z".to_string(),
+ })
+ .unwrap();
+ }
+ let mut reopened = TerminalCompactionJournal::open(&path).unwrap();
+ assert!(
+ reopened.pending("session-b").unwrap().is_some(),
+ "the record appended after the reopened journal must survive a reload"
+ );
+ assert!(reopened.pending("session-a").unwrap().is_some());
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn corrupt_complete_terminal_identifiers_are_preserved_before_recovery_fails() {
+ for newline_terminated in [true, false] {
+ let dir = std::env::temp_dir().join(format!("pa-comp-sup-{}", uuid::Uuid::new_v4()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join("compaction-supervision.jsonl");
+ let mut journal = TerminalCompactionJournal::open(&path).unwrap();
+ journal
+ .declare(TerminalCompactionRecord {
+ version: 1,
+ r#type: TERMINAL_COMPACTION_RECORD_TYPE.to_string(),
+ active_session_id: "badXsession".to_string(),
+ session_file: None,
+ reason: "threshold".to_string(),
+ declared_at: "2026-10-11T00:00:00Z".to_string(),
+ })
+ .unwrap();
+ drop(journal);
+ let mut bytes = std::fs::read(&path).unwrap();
+ let marker = b"badXsession";
+ let offset = bytes
+ .windows(marker.len())
+ .position(|part| part == marker)
+ .unwrap();
+ bytes[offset + 3] = 0xff;
+ if newline_terminated {
+ bytes.extend_from_slice(br#"{"activeSessionId":"torn"#);
+ } else {
+ assert_eq!(bytes.pop(), Some(b'\n'));
+ }
+ std::fs::write(&path, &bytes).unwrap();
+ assert!(TerminalCompactionJournal::open(&path).is_err());
+ assert_eq!(std::fs::read(&path).unwrap(), bytes);
+ std::fs::remove_dir_all(dir).unwrap();
+ }
+ }
+
+ #[test]
+ fn a_competing_terminal_writer_cannot_replace_an_active_append() {
+ use std::io::Write;
+ let dir = std::env::temp_dir().join(format!("pa-comp-sup-{}", uuid::Uuid::new_v4()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join("compaction-supervision.jsonl");
+ let first = TerminalCompactionRecord {
+ version: 1,
+ r#type: TERMINAL_COMPACTION_RECORD_TYPE.to_string(),
+ active_session_id: "session-a".to_string(),
+ session_file: None,
+ reason: "threshold".to_string(),
+ declared_at: "2026-10-11T00:00:00Z".to_string(),
+ };
+ let second = TerminalCompactionRecord {
+ active_session_id: "session-b".to_string(),
+ declared_at: "2026-10-11T00:00:01Z".to_string(),
+ ..first.clone()
+ };
+ let mut owner = TerminalCompactionJournal::open(&path).unwrap();
+ owner.declare(first.clone()).unwrap();
+ let mut bytes = serde_json::to_vec(&second).unwrap();
+ bytes.push(b'\n');
+ let split = bytes.len() / 2;
+ let mut active_append = std::fs::OpenOptions::new()
+ .append(true)
+ .open(&path)
+ .unwrap();
+ active_append.write_all(&bytes[..split]).unwrap();
+ let expected = std::fs::read(&path).unwrap();
+ let competing = TerminalCompactionJournal::open(&path);
+ assert_eq!(std::fs::read(&path).unwrap(), expected);
+ assert!(
+ competing.is_err(),
+ "a rival writer must not acquire the active journal"
+ );
+ active_append.write_all(&bytes[split..]).unwrap();
+ active_append.sync_all().unwrap();
+ drop(active_append);
+ drop(owner);
+ let mut reopened = TerminalCompactionJournal::open(&path).unwrap();
+ assert_eq!(reopened.pending("session-a").unwrap(), Some(&first));
+ assert_eq!(reopened.pending("session-b").unwrap(), Some(&second));
+ drop(reopened);
+ std::fs::remove_dir_all(dir).unwrap();
+ }
+
+ #[cfg(unix)]
+ #[tokio::test]
+ async fn a_competing_supervisor_does_not_heal_before_socket_admission() {
+ use std::io::Write;
+ let dir = std::path::PathBuf::from("/tmp").join(format!("pa-c-{}", uuid::Uuid::new_v4()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let options = crate::supervisor::SupervisorOptions {
+ socket_path: dir.join("supervisor.sock"),
+ agent_dir: dir.join("agent"),
+ };
+ let descriptor_dir =
+ crate::descriptor::descriptor_dir(&options.agent_dir, &options.socket_path);
+ std::fs::create_dir_all(&descriptor_dir).unwrap();
+ let path = descriptor_dir.join("compaction-supervision.jsonl");
+ let first = TerminalCompactionRecord {
+ version: 1,
+ r#type: TERMINAL_COMPACTION_RECORD_TYPE.to_string(),
+ active_session_id: "session-a".to_string(),
+ session_file: None,
+ reason: "threshold".to_string(),
+ declared_at: "2026-10-11T00:00:00Z".to_string(),
+ };
+ let second = TerminalCompactionRecord {
+ active_session_id: "session-b".to_string(),
+ declared_at: "2026-10-11T00:00:01Z".to_string(),
+ ..first.clone()
+ };
+ let mut original = serde_json::to_vec(&first).unwrap();
+ original.push(b'\n');
+ std::fs::write(&path, original).unwrap();
+ let listener = pa_types::platform::transport::bind_transport(&options.socket_path)
+ .await
+ .unwrap();
+ // A legacy writer does not take the proposed journal sidecar lock.
+ let mut active_append = std::fs::OpenOptions::new()
+ .append(true)
+ .open(&path)
+ .unwrap();
+ let mut bytes = serde_json::to_vec(&second).unwrap();
+ bytes.push(b'\n');
+ let split = bytes.len() / 2;
+ active_append.write_all(&bytes[..split]).unwrap();
+ let expected = std::fs::read(&path).unwrap();
+ let contender = crate::supervisor::Supervisor::new(options).unwrap();
+ assert_eq!(std::fs::read(&path).unwrap(), expected);
+ assert!(std::sync::Arc::new(contender).run().await.is_err());
+ assert_eq!(std::fs::read(&path).unwrap(), expected);
+ active_append.write_all(&bytes[split..]).unwrap();
+ active_append.sync_all().unwrap();
+ drop(active_append);
+ let mut reopened = TerminalCompactionJournal::open(&path).unwrap();
+ assert_eq!(reopened.pending("session-a").unwrap(), Some(&first));
+ assert_eq!(reopened.pending("session-b").unwrap(), Some(&second));
+ drop(reopened);
+ drop(listener);
+ std::fs::remove_dir_all(dir).unwrap();
+ }
}
diff --git a/crates/pa-daemon/src/journal.rs b/crates/pa-daemon/src/journal.rs
index 1f308927eb..d081ec9732 100644
--- a/crates/pa-daemon/src/journal.rs
+++ b/crates/pa-daemon/src/journal.rs
@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
use std::fs::{self, File, OpenOptions};
-use std::io::{BufWriter, Write};
+use std::io::{BufWriter, Read, Seek, SeekFrom, Write};
use std::path::Path;
pub(crate) const RECOVERY_JOURNAL_SUFFIX: &str = ".recovery.jsonl";
@@ -84,6 +84,87 @@ pub(crate) fn rewrite_records(path: &Path, records: &[Value], finalize: Finalize
Ok(())
}
+// Complete records must decode exactly before recovery can replace their history.
+// Only an unterminated final JSON fragment may end with incomplete UTF-8.
+pub(crate) fn decode_journal_lines<'a>(bytes: &'a [u8], path: &Path) -> Result<Vec<&'a str>> {
+ let mut lines = Vec::new();
+ let mut records = bytes.split(|byte| *byte == b'\n').peekable();
+ while let Some(raw) = records.next() {
+ match std::str::from_utf8(raw) {
+ Ok(line) => lines.push(line),
+ Err(error) => {
+ let incomplete_final = records.peek().is_none()
+ && !bytes.ends_with(b"\n")
+ && serde_json::from_slice::<Value>(raw).is_err_and(|error| error.is_eof());
+ if !incomplete_final {
+ return Err(error)
+ .with_context(|| format!("invalid UTF-8 in journal {}", path.display()));
+ }
+ }
+ }
+ }
+ Ok(lines)
+}
+
+pub(crate) struct JournalWriteOwner {
+ file: File,
+}
+
+impl Drop for JournalWriteOwner {
+ fn drop(&mut self) {
+ // Closing one descriptor can leave the lock held by an inherited copy.
+ if let Err(error) = pa_core::platform::unlock_file(&self.file) {
+ // Drop has no error channel; report without panicking if stderr is unavailable.
+ let _ = writeln!(
+ std::io::stderr(),
+ "Prime Agent: failed to release journal ownership: {error}"
+ );
+ }
+ }
+}
+
+// Keep the sidecar inode stable: unlinking can admit a second ownership inode.
+// Empty sidecars remain after retirement; directory entries/inodes can accumulate.
+// Explicit unlock is attempted on guard drop; a failure leaves close as fallback.
+pub(crate) fn journal_write_owner(path: &Path) -> Result<JournalWriteOwner> {
+ let mut ownership_path = path.as_os_str().to_os_string();
+ ownership_path.push(".writer-lock");
+ let file = OpenOptions::new()
+ .create(true)
+ .read(true)
+ .write(true)
+ .truncate(false)
+ .open(std::path::PathBuf::from(ownership_path))
+ .with_context(|| format!("open journal ownership {}", path.display()))?;
+ pa_core::platform::try_lock_exclusive(&file)
+ .with_context(|| format!("acquire journal ownership {}", path.display()))?;
+ Ok(JournalWriteOwner { file })
+}
+
+pub(crate) fn tail_is_torn(path: &Path) -> Result<bool> {
+ let mut file = match File::open(path) {
+ Ok(file) => file,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
+ Err(error) => {
+ return Err(error).with_context(|| format!("open journal tail {}", path.display()));
+ }
+ };
+ if file
+ .metadata()
+ .with_context(|| format!("stat journal tail {}", path.display()))?
+ .len()
+ == 0
+ {
+ return Ok(false);
+ }
+ file.seek(SeekFrom::End(-1))
+ .with_context(|| format!("seek journal tail {}", path.display()))?;
+ let mut tail = [0u8];
+ file.read_exact(&mut tail)
+ .with_context(|| format!("read journal tail {}", path.display()))?;
+ Ok(tail[0] != b'\n')
+}
+
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct WorkerRecoveryRecord {
@@ -98,10 +179,14 @@ pub struct WorkerRecoveryRecord {
fn parse_worker_records(path: &Path) -> Result<HashMap<String, WorkerRecoveryRecord>> {
let mut latest = HashMap::new();
- let Ok(content) = fs::read_to_string(path) else {
- return Ok(latest);
+ let bytes = match fs::read(path) {
+ Ok(bytes) => bytes,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(latest),
+ Err(error) => {
+ return Err(error).with_context(|| format!("read journal {}", path.display()));
+ }
};
- for line in content.lines() {
+ for line in decode_journal_lines(&bytes, path)? {
if line.is_empty() {
continue;
}
@@ -172,6 +257,7 @@ pub struct WorkerQueueSnapshotRecord {
/// Latest busy/operation per active session, plus the latest queue
/// snapshot per session.
pub struct WorkerRecoveryJournal {
+ _write_owner: JournalWriteOwner,
path: std::path::PathBuf,
latest: HashMap<String, WorkerRecoveryRecord>,
queue_snapshots: HashMap<String, WorkerQueueSnapshotRecord>,
@@ -182,18 +268,28 @@ impl WorkerRecoveryJournal {
///
/// # Errors
///
- /// Returns an error when the parent directory cannot be created, or
- /// the queue-snapshot pass cannot read an existing journal.
+ /// Returns an error when write ownership cannot be acquired, the parent directory cannot
+ /// be created, either
+ /// record pass cannot read an existing journal, or the
+ /// torn-tail heal cannot rewrite it.
pub fn open(path: &Path) -> Result<Self> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
+ let write_owner = journal_write_owner(path)?;
let queue_snapshots = parse_queue_snapshot_records(path)?;
- Ok(WorkerRecoveryJournal {
+ let journal = WorkerRecoveryJournal {
+ _write_owner: write_owner,
path: path.to_path_buf(),
latest: parse_worker_records(path)?,
queue_snapshots,
- })
+ };
+ if tail_is_torn(path)? {
+ // Recovery can still contain busy verdicts and queued prompts.
+ // Sync the replacement before it takes over their durable history.
+ journal.compact(Finalize::Synced)?;
+ }
+ Ok(journal)
}
/// Read the latest worker record per active session straight from a
@@ -201,8 +297,8 @@ impl WorkerRecoveryJournal {
///
/// # Errors
///
- /// Never errors: a missing or unreadable journal reads as an empty
- /// set (the `Result` wrapper keeps the reading seam uniform).
+ /// Returns an error when an existing journal cannot be read; a missing
+ /// journal reads as an empty set.
pub fn read_latest(path: &Path) -> Result<Vec<WorkerRecoveryRecord>> {
Ok(parse_worker_records(path)?.into_values().collect())
}
@@ -290,7 +386,7 @@ impl WorkerRecoveryJournal {
// after `set`); checking before it, a single-session journal never
// compacted.
if self.latest.values().all(|entry| !entry.busy) {
- self.compact()?;
+ self.compact(Finalize::Bare)?;
}
Ok(())
}
@@ -382,7 +478,7 @@ impl WorkerRecoveryJournal {
// TS parity (same post-insert check as `record`): the
// compaction fires on the all-idle map including the verdict.
if self.latest.values().all(|entry| !entry.busy) {
- self.compact()?;
+ self.compact(Finalize::Bare)?;
}
}
Ok(())
@@ -415,7 +511,7 @@ impl WorkerRecoveryJournal {
.map(|record| (record.steering, record.follow_up)))
}
- fn compact(&self) -> Result<()> {
+ fn compact(&self, finalize: Finalize) -> Result<()> {
let mut records: Vec<Value> = self
.latest
.values()
@@ -427,7 +523,7 @@ impl WorkerRecoveryJournal {
.map(serde_json::to_value)
.collect::<std::result::Result<_, _>>()?;
records.extend(snapshots);
- rewrite_records(&self.path, &records, Finalize::Bare)
+ rewrite_records(&self.path, &records, finalize)
}
}
@@ -438,14 +534,14 @@ const QUEUE_SNAPSHOT_VERSION: u32 = 2;
fn parse_queue_snapshot_records(path: &Path) -> Result<HashMap<String, WorkerQueueSnapshotRecord>> {
let mut latest: HashMap<String, WorkerQueueSnapshotRecord> = HashMap::new();
- let contents = match fs::read_to_string(path) {
- Ok(contents) => contents,
+ let bytes = match fs::read(path) {
+ Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(latest),
Err(error) => {
return Err(error).with_context(|| format!("read journal {}", path.display()))
}
};
- for line in contents.split('\n').filter(|line| !line.is_empty()) {
+ for line in decode_journal_lines(&bytes, path)? {
let Ok(record) = serde_json::from_str::<Value>(line) else {
continue;
};
@@ -513,6 +609,20 @@ mod tests {
dir.join(name)
}
+ #[test]
+ fn worker_journal_read_failures_are_not_empty_recovery_state() {
+ let path = temp_path("unreadable.jsonl");
+ fs::create_dir_all(&path).unwrap();
+ let sentinel = path.join("preserve");
+ fs::write(&sentinel, b"original data").unwrap();
+
+ assert!(tail_is_torn(&path).is_err());
+ assert!(parse_worker_records(&path).is_err());
+ assert!(WorkerRecoveryJournal::open(&path).is_err());
+ assert_eq!(fs::read(&sentinel).unwrap(), b"original data");
+ fs::remove_dir_all(path.parent().unwrap()).unwrap();
+ }
+
#[test]
fn worker_journal_keeps_latest_per_session() {
let path = temp_path("worker.recovery.jsonl");
@@ -679,6 +789,7 @@ mod tests {
let record: Value = serde_json::from_str(lines[0]).unwrap();
assert_eq!(record["busy"], false);
assert_eq!(record["operation"], "turn_end");
+ drop(journal);
let reopened = WorkerRecoveryJournal::open(&path).unwrap();
let latest = reopened.get_latest();
assert_eq!(latest.len(), 1);
@@ -746,6 +857,7 @@ mod tests {
// the compact keeps the LATEST snapshot per session: the
// settle's (empty) lanes, not the admission's parked row.
assert_eq!(snapshot["steering"].as_array().map(Vec::len), Some(0));
+ drop(journal);
let reopened = WorkerRecoveryJournal::open(&path).unwrap();
assert!(!WorkerRecoveryJournal::read_interrupted(&path));
let restored = reopened.latest_queue_snapshot("s1").unwrap();
@@ -782,4 +894,257 @@ mod tests {
assert!(!WorkerRecoveryJournal::read_interrupted(&path));
let _ = fs::remove_dir_all(path.parent().unwrap());
}
+
+ #[test]
+ fn worker_journal_open_skips_torn_utf8_tail() {
+ let path = temp_path("torn-utf8-tail.recovery.jsonl");
+ let item = WorkerQueueItemRecord {
+ message: "steer ünïcode".to_string(),
+ priority: Some(crate::worker::QueuePriority::Human),
+ preview: Some("preview".to_string()),
+ custom_message: None,
+ queue_key: None,
+ queue_visible: true,
+ policy: queue_policy_default(),
+ };
+ let mut journal = WorkerRecoveryJournal::open(&path).unwrap();
+ journal
+ .record_queue_checkpoint(
+ "s1",
+ "sess1",
+ Some("/a.jsonl"),
+ true,
+ "prompt_accepted",
+ std::slice::from_ref(&item),
+ &[],
+ )
+ .unwrap();
+ let whole = serde_json::to_string(&WorkerRecoveryRecord {
+ active_session_id: "s2".to_string(),
+ session_id: "sess2".to_string(),
+ session_file: None,
+ busy: false,
+ operation: "shütdown".to_string(),
+ recorded_at: "2026-10-07T00:00:00Z".to_string(),
+ })
+ .unwrap();
+ let torn = &whole.as_bytes()[..=whole.find('ü').unwrap()];
+ assert!(std::str::from_utf8(torn).is_err());
+ let mut file = OpenOptions::new().append(true).open(&path).unwrap();
+ file.write_all(torn).unwrap();
+ drop(file);
+
+ drop(journal);
+ let mut healed = WorkerRecoveryJournal::open(&path).unwrap();
+ // Reload after a fresh append so the assertions exercise the healed
+ // file, rather than only the state loaded before the rewrite.
+ healed
+ .record("s1", "sess1", Some("/a.jsonl"), true, "resumed")
+ .unwrap();
+ drop(healed);
+ let reopened = WorkerRecoveryJournal::open(&path).unwrap();
+ assert!(WorkerRecoveryJournal::read_interrupted(&path));
+ let latest = reopened.get_latest();
+ assert_eq!(latest.len(), 1);
+ assert_eq!(latest[0].active_session_id, "s1");
+ assert!(latest[0].busy);
+ let restored = reopened.latest_queue_snapshot("s1").unwrap();
+ assert_eq!(restored.0, vec![item]);
+ assert!(restored.1.is_empty());
+ let _ = fs::remove_dir_all(path.parent().unwrap());
+ }
+
+ #[test]
+ fn worker_journal_open_heals_torn_ascii_tail() {
+ let path = temp_path("torn-ascii-tail.recovery.jsonl");
+ let mut journal = WorkerRecoveryJournal::open(&path).unwrap();
+ journal
+ .record("s1", "sess1", Some("/a.jsonl"), true, "prompt_accepted")
+ .unwrap();
+ let mut file = OpenOptions::new().append(true).open(&path).unwrap();
+ file.write_all(br#"{"activeSessionId":"s3","sessionId":"ses"#)
+ .unwrap();
+ drop(file);
+
+ drop(journal);
+ let mut journal = WorkerRecoveryJournal::open(&path).unwrap();
+ journal.record("s2", "sess2", None, false, "ready").unwrap();
+ drop(journal);
+ let reopened = WorkerRecoveryJournal::open(&path).unwrap();
+ assert!(WorkerRecoveryJournal::read_interrupted(&path));
+ assert_eq!(
+ reopened
+ .get_latest()
+ .iter()
+ .filter(|record| record.active_session_id == "s2")
+ .count(),
+ 1,
+ "the record appended after the reopened journal must survive a reload"
+ );
+ let _ = fs::remove_dir_all(path.parent().unwrap());
+ }
+
+ #[test]
+ fn corrupt_complete_queue_bytes_are_preserved_before_recovery_fails() {
+ for newline_terminated in [true, false] {
+ let path = temp_path("corrupt-complete-queue.recovery.jsonl");
+ let mut journal = WorkerRecoveryJournal::open(&path).unwrap();
+ let item = WorkerQueueItemRecord {
+ message: "safeXinstruction".to_string(),
+ priority: None,
+ preview: None,
+ custom_message: None,
+ queue_key: None,
+ queue_visible: true,
+ policy: queue_policy_default(),
+ };
+ journal.record_queue_snapshot("s1", &[item], &[]).unwrap();
+ drop(journal);
+ let mut bytes = fs::read(&path).unwrap();
+ let marker = b"safeXinstruction";
+ let offset = bytes
+ .windows(marker.len())
+ .position(|part| part == marker)
+ .unwrap();
+ bytes[offset + 4] = 0xff;
+ if newline_terminated {
+ bytes.extend_from_slice(br#"{"activeSessionId":"torn"#);
+ } else {
+ assert_eq!(bytes.pop(), Some(b'\n'));
+ }
+ fs::write(&path, &bytes).unwrap();
+ let result = WorkerRecoveryJournal::open(&path);
+ assert_eq!(fs::read(&path).unwrap(), bytes);
+ assert!(
+ result.is_err(),
+ "complete corrupt queued prompts must block recovery"
+ );
+ fs::remove_dir_all(path.parent().unwrap()).unwrap();
+ }
+ }
+
+ #[test]
+ fn corrupt_complete_worker_identifiers_are_preserved_before_recovery_fails() {
+ for newline_terminated in [true, false] {
+ let path = temp_path("corrupt-complete-verdict.recovery.jsonl");
+ let mut journal = WorkerRecoveryJournal::open(&path).unwrap();
+ journal
+ .record("badXsession", "sess1", None, true, "prompt_accepted")
+ .unwrap();
+ drop(journal);
+ let mut bytes = fs::read(&path).unwrap();
+ let marker = b"badXsession";
+ let offset = bytes
+ .windows(marker.len())
+ .position(|part| part == marker)
+ .unwrap();
+ bytes[offset + 3] = 0xff;
+ if newline_terminated {
+ bytes.extend_from_slice(br#"{"activeSessionId":"torn"#);
+ } else {
+ assert_eq!(bytes.pop(), Some(b'\n'));
+ }
+ fs::write(&path, &bytes).unwrap();
+ assert!(WorkerRecoveryJournal::read_latest(&path).is_err());
+ assert!(WorkerRecoveryJournal::open(&path).is_err());
+ assert_eq!(fs::read(&path).unwrap(), bytes);
+ fs::remove_dir_all(path.parent().unwrap()).unwrap();
+ }
+ }
+
+ #[test]
+ fn a_complete_worker_record_without_newline_survives_healing() {
+ let path = temp_path("complete-eof.recovery.jsonl");
+ let mut journal = WorkerRecoveryJournal::open(&path).unwrap();
+ journal
+ .record("s1", "sess1", None, true, "prompt_accepted")
+ .unwrap();
+ let expected = journal.get_latest();
+ drop(journal);
+ let mut bytes = fs::read(&path).unwrap();
+ assert_eq!(bytes.pop(), Some(b'\n'));
+ fs::write(&path, bytes).unwrap();
+ let reopened = WorkerRecoveryJournal::open(&path).unwrap();
+ assert_eq!(reopened.get_latest(), expected);
+ assert_eq!(fs::read(&path).unwrap().last(), Some(&b'\n'));
+ drop(reopened);
+ fs::remove_dir_all(path.parent().unwrap()).unwrap();
+ }
+
+ #[test]
+ fn competing_worker_writers_cannot_replace_an_active_append() {
+ for contender in ["open", "settle"] {
+ let path = temp_path("active-append.recovery.jsonl");
+ let mut owner = WorkerRecoveryJournal::open(&path).unwrap();
+ owner
+ .record("s1", "sess1", None, true, "prompt_accepted")
+ .unwrap();
+ let first = owner.get_latest().pop().unwrap();
+ let record = WorkerRecoveryRecord {
+ active_session_id: "s2".to_string(),
+ session_id: "sess2".to_string(),
+ session_file: None,
+ busy: true,
+ operation: "prompt_accepted".to_string(),
+ recorded_at: "2026-10-11T00:00:01Z".to_string(),
+ };
+ let mut bytes = serde_json::to_vec(&record).unwrap();
+ bytes.push(b'\n');
+ let split = bytes.len() / 2;
+ let mut active_append = OpenOptions::new().append(true).open(&path).unwrap();
+ active_append.write_all(&bytes[..split]).unwrap();
+ let expected = fs::read(&path).unwrap();
+ let result = match contender {
+ "open" => WorkerRecoveryJournal::open(&path).map(drop),
+ "settle" => WorkerRecoveryJournal::settle_busy_records(&path, "worker_gave_up"),
+ _ => unreachable!("known contender"),
+ };
+ assert_eq!(fs::read(&path).unwrap(), expected);
+ assert!(
+ result.is_err(),
+ "a rival writer must not acquire the active journal"
+ );
+ active_append.write_all(&bytes[split..]).unwrap();
+ active_append.sync_all().unwrap();
+ drop(active_append);
+ drop(owner);
+ let reopened = WorkerRecoveryJournal::open(&path).unwrap();
+ let mut latest = reopened.get_latest();
+ latest.sort_by(|left, right| left.active_session_id.cmp(&right.active_session_id));
+ assert_eq!(latest, vec![first, record]);
+ drop(reopened);
+ fs::remove_dir_all(path.parent().unwrap()).unwrap();
+ }
+ }
+
+ #[cfg(unix)]
+ #[test]
+ #[allow(clippy::used_underscore_binding)] // Intentionally duplicate the private lifetime guard to model an inherited descriptor.
+ fn dropping_a_journal_releases_ownership_while_a_duplicate_handle_survives() {
+ let path = temp_path("duplicate-owner.recovery.jsonl");
+ let mut owner = WorkerRecoveryJournal::open(&path).unwrap();
+ owner
+ .record("s1", "sess1", None, true, "prompt_accepted")
+ .unwrap();
+ let expected = owner.get_latest();
+ let inherited = owner._write_owner.file.try_clone().unwrap();
+ assert!(WorkerRecoveryJournal::open(&path).is_err());
+
+ drop(owner);
+ let successor = WorkerRecoveryJournal::open(&path)
+ .expect("logical owner drop releases its lock even while a duplicate handle survives");
+ assert_eq!(successor.get_latest(), expected);
+ assert!(WorkerRecoveryJournal::open(&path).is_err());
+
+ drop(inherited);
+ assert!(
+ WorkerRecoveryJournal::open(&path).is_err(),
+ "closing the old handle must not release the successor's ownership",
+ );
+ drop(successor);
+ let reopened = WorkerRecoveryJournal::open(&path).unwrap();
+ assert_eq!(reopened.get_latest(), expected);
+ drop(reopened);
+ fs::remove_dir_all(path.parent().unwrap()).unwrap();
+ }
}
diff --git a/crates/pa-daemon/src/supervisor.rs b/crates/pa-daemon/src/supervisor.rs
index 8f9aeb4dc1..c4200d1ee7 100644
--- a/crates/pa-daemon/src/supervisor.rs
+++ b/crates/pa-daemon/src/supervisor.rs
@@ -220,15 +220,15 @@ pub struct Supervisor {
/// when the worker could not answer; feeds the replacement-worker create replay,
/// cleared by a landed `compaction_end`.
pub(crate) compaction_journal:
- std::sync::Mutex<crate::compaction_supervision::TerminalCompactionJournal>,
+ std::sync::Mutex<Option<crate::compaction_supervision::TerminalCompactionJournal>>,
}
impl Supervisor {
- /// Build the supervisor: descriptor dir, persisted config, event channel, log, journal.
+ /// Build the supervisor: descriptor dir, persisted config, event channel, and log.
///
/// # Errors
///
- /// Returns an error when the descriptor dir, sessions dir, config, or journal cannot be opened.
+ /// Returns an error when the descriptor dir, sessions dir, or config cannot be opened.
pub fn new(options: SupervisorOptions) -> Result<Self> {
let descriptor_dir =
crate::descriptor::descriptor_dir(&options.agent_dir, &options.socket_path);
@@ -250,9 +250,6 @@ impl Supervisor {
&options.socket_path,
&options.agent_dir,
));
- let compaction_journal = crate::compaction_supervision::TerminalCompactionJournal::open(
- &descriptor_dir.join("compaction-supervision.jsonl"),
- )?;
Ok(Supervisor {
options,
descriptor_dir,
@@ -286,7 +283,7 @@ impl Supervisor {
passive_scan_gate: tokio::sync::Mutex::new(()),
passive_scan_pending: std::sync::atomic::AtomicBool::new(false),
passive_catalog_epoch: std::sync::atomic::AtomicU64::new(0),
- compaction_journal: std::sync::Mutex::new(compaction_journal),
+ compaction_journal: std::sync::Mutex::new(None),
})
}
@@ -295,7 +292,8 @@ impl Supervisor {
/// # Errors
///
/// Returns an error when the socket path cannot be prepared (already in use), the
- /// socket cannot be bound, or the accept loop exhausts its give-up budget.
+ /// socket cannot be bound, the recovery journal cannot be opened, or the accept loop
+ /// exhausts its give-up budget.
///
/// # Panics
///
@@ -363,6 +361,22 @@ impl Supervisor {
*self.bound_socket_identity.lock().unwrap() =
socket::socket_identity(&self.options.socket_path);
socket::restrict_socket_path(&self.options.socket_path);
+ match crate::compaction_supervision::TerminalCompactionJournal::open(
+ &self.descriptor_dir.join("compaction-supervision.jsonl"),
+ ) {
+ Ok(journal) => {
+ *self
+ .compaction_journal
+ .lock()
+ .expect("compaction journal lock") = Some(journal);
+ }
+ Err(error) => {
+ drop(listener);
+ let identity = self.bound_socket_identity.lock().unwrap().clone();
+ socket::cleanup_socket_path_after_close(&self.options.socket_path, identity);
+ return Err(error);
+ }
+ }
self.log
.append(&format!("supervisor started pid {}", std::process::id()));
match open_file_limit {
diff --git a/crates/pa-daemon/src/supervisor/supervision.rs b/crates/pa-daemon/src/supervisor/supervision.rs
index f689fc33ea..1f8c2c2556 100644
--- a/crates/pa-daemon/src/supervisor/supervision.rs
+++ b/crates/pa-daemon/src/supervisor/supervision.rs
@@ -166,17 +166,30 @@ impl Supervisor {
descriptor.lifecycle = DaemonWorkerLifecycle::Failed;
descriptor.last_failure_at = Some(util::now_iso());
let journal_path = descriptor.recovery_journal_path.clone();
+ let owner_pid = descriptor.pid as u32;
+ let owner_start_id = descriptor.process_start_id.clone();
let _ = persist_worker(&resident.descriptor_path, &descriptor);
drop(descriptor);
// The give-up settles its own revival evidence: the journal's busy records
// are what a later boot reads as "interrupted live work" — the give-up
// verdict must outlive them.
- if let Err(error) = crate::journal::WorkerRecoveryJournal::settle_busy_records(
- std::path::Path::new(&journal_path),
- "worker_gave_up",
+ if super::worker_lifecycle::recovery_journal_owner_is_gone(
+ is_process_alive(owner_pid).ok(),
+ owner_start_id.as_deref(),
+ crate::lease::get_process_start_id(owner_pid).as_deref(),
) {
+ if let Err(error) = crate::journal::WorkerRecoveryJournal::settle_busy_records(
+ std::path::Path::new(&journal_path),
+ "worker_gave_up",
+ ) {
+ self.log_line(&format!(
+ "session worker {} give-up journal settle failed: {error:#}",
+ resident.worker_id
+ ));
+ }
+ } else {
self.log_line(&format!(
- "session worker {} give-up journal settle failed: {error:#}",
+ "session worker {} give-up journal preserved: owner death is unproven",
resident.worker_id
));
}
@@ -298,12 +311,12 @@ impl Supervisor {
// discloses the aborted run in the rebuilt transcript, and the record is
// consumed by the reply.
let pending = {
- let mut journal = self
- .compaction_journal
- .lock()
- .expect("compaction journal lock");
- match journal.pending(&descriptor.root_active_session_id) {
- Ok(pending) => pending.cloned(),
+ match self.with_compaction_journal(|journal| {
+ Ok(journal
+ .pending(&descriptor.root_active_session_id)?
+ .cloned())
+ }) {
+ Ok(pending) => pending,
Err(error) => {
self.log_line(&format!(
"terminal compaction journal retry failed for {}: {error:#}",
@@ -393,11 +406,8 @@ impl Supervisor {
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
if persisted {
- if let Err(error) = self
- .compaction_journal
- .lock()
- .expect("compaction journal lock")
- .consume(&root_active_session_id)
+ if let Err(error) =
+ self.with_compaction_journal(|journal| journal.consume(&root_active_session_id))
{
self.log_line(&format!(
"terminal compaction journal consume failed for {root_active_session_id}: {error:#}"
@@ -718,19 +728,17 @@ impl Supervisor {
}
}
Some("compaction_end") => {
- let clear_error = {
- let mut journal = reader_supervisor
- .compaction_journal
- .lock()
- .expect("compaction journal lock");
- reader_resident.compaction.observe_end();
- active_session_id.as_deref().and_then(|active_session_id| {
- // A failed clear keeps the record pending
- // (write-before-forget, so memory and disk agree);
- // retried by the next forwarded end.
- journal.clear(active_session_id).err()
+ let clear_error = reader_supervisor
+ .with_compaction_journal(|journal| {
+ reader_resident.compaction.observe_end();
+ if let Some(active_session_id) =
+ active_session_id.as_deref()
+ {
+ journal.clear(active_session_id)?;
+ }
+ Ok(())
})
- };
+ .err();
if let Some(error) = clear_error {
reader_supervisor.log_line(&format!(
"terminal compaction journal clear failed for {active_session_id:?}: {error:#}"
diff --git a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs b/crates/pa-daemon/src/supervisor/worker_lifecycle.rs
index ff3247db20..3da13b383b 100644
--- a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs
+++ b/crates/pa-daemon/src/supervisor/worker_lifecycle.rs
@@ -797,7 +797,7 @@ impl Supervisor {
/// A dead process or a verified replacement identity proves the old writer
/// is gone. Failed probes and missing live identities preserve the journal.
-fn recovery_journal_owner_is_gone(
+pub(super) fn recovery_journal_owner_is_gone(
process_alive: Option<bool>,
expected_start_id: Option<&str>,
current_start_id: Option<&str>,
diff --git a/crates/pa-daemon/src/worker/connection.rs b/crates/pa-daemon/src/worker/connection.rs
index c39ceefc3f..2ef3d3515d 100644
--- a/crates/pa-daemon/src/worker/connection.rs
+++ b/crates/pa-daemon/src/worker/connection.rs
@@ -210,14 +210,6 @@ impl Worker {
///
/// Panics when the recovery mutex is poisoned.
pub async fn serve(self: Arc<Self>) -> Result<()> {
- *self.recovery.lock().unwrap() = Some(WorkerRecoveryJournal::open(
- &self.config.recovery_journal_path,
- )?);
- // A worker spawned under a supervisor arms the orphan-exit monitor:
- // nobody else reaps it if the supervisor dies without a graceful stop.
- if !self.config.supervisor_socket_path.as_os_str().is_empty() {
- crate::supervisor_lost::start(self.clone());
- }
if let Err(error) = crate::socket::prepare_socket_path(&self.config.socket_path).await {
// The no-listener confirmation of the close handshake: an
// exit path already parked on the confirmation proceeds (its
@@ -257,6 +249,21 @@ impl Worker {
*self.bound_socket_identity.lock().unwrap() =
crate::socket::socket_identity(&self.config.socket_path);
crate::socket::restrict_socket_path(&self.config.socket_path);
+ match WorkerRecoveryJournal::open(&self.config.recovery_journal_path) {
+ Ok(journal) => *self.recovery.lock().unwrap() = Some(journal),
+ Err(error) => {
+ drop(listener);
+ let identity = self.bound_socket_identity.lock().unwrap().clone();
+ crate::socket::cleanup_socket_path_after_close(&self.config.socket_path, identity);
+ self.listener_closed.notify_one();
+ return Err(error);
+ }
+ }
+ // No journal snapshot is loaded before endpoint and write ownership.
+ if !self.config.supervisor_socket_path.as_os_str().is_empty() {
+ crate::supervisor_lost::start(self.clone());
+ }
+
// This loop owns the bound listener and hands its close to the
// exit paths (TS daemon-mode.ts:8011-8018 awaits `server.close()`
// FIRST and cleans the socket path after): an exiting path
diff --git a/crates/pa-daemon/src/worker/tests/queue_tests.rs b/crates/pa-daemon/src/worker/tests/queue_tests.rs
index eb0f7421d6..e93036ef9f 100644
--- a/crates/pa-daemon/src/worker/tests/queue_tests.rs
+++ b/crates/pa-daemon/src/worker/tests/queue_tests.rs
@@ -293,6 +293,7 @@ fn legacy_queue_record_priority_defaults_by_row_and_keeps_order() {
journal
.record_queue_snapshot("legacy", &[machine, human, future], &[])
.unwrap();
+ drop(journal);
let reopened = WorkerRecoveryJournal::open(&path).unwrap();
let (lane, _) = restore_queue_snapshot(&reopened, "legacy");
assert_eq!(
@@ -352,6 +353,7 @@ fn queue_snapshot_round_trips_through_the_recovery_journal() {
)
.unwrap();
// A reopen (respawned worker) reads the latest snapshot per session.
+ drop(journal);
let reloaded = WorkerRecoveryJournal::open(&journal_path).unwrap();
let (steering, follow_up) = restore_queue_snapshot(&reloaded, "session-a");
assert_eq!(steering.len(), 1);
@@ -366,10 +368,12 @@ fn queue_snapshot_round_trips_through_the_recovery_journal() {
assert_eq!(follow_up[0].priority, QueuePriority::Human);
// Compaction (triggered by an all-idle record) keeps the snapshot
// with its full rows.
+ drop(reloaded);
let mut compacting = WorkerRecoveryJournal::open(&journal_path).unwrap();
compacting
.record("session-a", "s1", None, false, "idle")
.unwrap();
+ drop(compacting);
let compacted = WorkerRecoveryJournal::open(&journal_path).unwrap();
let (steering, _) = restore_queue_snapshot(&compacted, "session-a");
assert_eq!(steering.len(), 1);
diff --git a/crates/pa-daemon/src/worker/update_snapshot_tests.rs b/crates/pa-daemon/src/worker/update_snapshot_tests.rs
index d4c58300f3..4b32ccf4b6 100644
--- a/crates/pa-daemon/src/worker/update_snapshot_tests.rs
+++ b/crates/pa-daemon/src/worker/update_snapshot_tests.rs
@@ -575,8 +575,19 @@ async fn restored_lane_rows_keep_the_child_status_provenance() {
follow_up: Vec::new(),
},
);
- let journal = WorkerRecoveryJournal::open(&worker.config.recovery_journal_path).unwrap();
- let (steering, follow_up) = restore_queue_snapshot(&journal, "target-session");
+ let (steering, follow_up) = {
+ let recovery = worker.recovery.lock().unwrap();
+ let journal = recovery.as_ref().unwrap();
+ assert_eq!(
+ WorkerRecoveryJournal::read_queue_snapshot(
+ &worker.config.recovery_journal_path,
+ "target-session",
+ )
+ .unwrap(),
+ journal.latest_queue_snapshot("target-session"),
+ );
+ restore_queue_snapshot(journal, "target-session")
+ };
assert_eq!(steering.len(), 1);
assert!(follow_up.is_empty());
assert!(