A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
restore continuous background trace uploads (#3415)
kt2a16de9060fd
diff --git a/crates/pa-core/README.md b/crates/pa-core/README.mdindex c249f76779..bdca2643cb 100644--- a/crates/pa-core/README.md+++ b/crates/pa-core/README.md@@ -78,3 +78,30 @@ No provider HTTP (pa-ai), no loop policy (pa-agent), no daemon supervision (pa-d## Depends onpa-types, pa-ai, pa-agent (one-way).+++Automatic trace delivery (`agent_traces::ContinuousTraceUpload`) owns consent-gated+persist scheduling and startup outbox recovery alongside the existing manual uploader.+Hosts reuse `load_settings` to capture consent before their existing settings parse,+attach `persisted` after successful disk writes, and own the returned installation+for the session lifetime. Replacements use `rebind` with the effective cwd; forks+use `forked`. The installation never drains on drop. A detached process-wide runtime+owns all transcript reads, scans, settings reloads, networking and retries; a maximum+of 256 registered controllers and four deliveries bounds resident work. Admission+returns `None` at capacity and logs rejection; hosts attach no hook in that case.+Retired descriptors free capacity only after background recovery acknowledges them.+Session replacement transfers its existing slot and cancels the predecessor; forks+require a distinct slot. Replacement retains a bounded retirement flag until the+background recovery snapshot acknowledges its exact controller identity. The synchronous+persist seam checks the cached consent snapshot and performs an exists/create+pending-marker write; consent metadata reads and reloads stay in the worker after+the host captures its initial settings generation; it does not fsync the marker (matching TS v0.9.8 process-crash+recovery rather than promising power-loss durability). A short nonblocking mutation lease prevents cursor/prune races; contention+records one stable fallback marker per session, which is retained conservatively+until a future sender has delivered it. Its cost must be measured+separately; startup/paint parity is not established by scheduler unit tests.+Active requests monitor consent changes every second on the worker and cancel on+revocation/errors; bytes already sent cannot be recalled. Recovery belongs to all+live hosts sharing the agent directory and cancels when the last host disappears.+Catch-up uses at most three delivery cycles per pending entry, honors Retry-After,+and releases upload capacity during retry waits; exhaustion retains durable intent.diff --git a/crates/pa-core/src/agent_traces.rs b/crates/pa-core/src/agent_traces.rsindex fd6a48356c..d4016a3c47 100644--- a/crates/pa-core/src/agent_traces.rs+++ b/crates/pa-core/src/agent_traces.rs@@ -25,6 +25,9 @@ pub use upload_all::{TraceUploadAllOptions,};+mod continuous;+pub use continuous::{ContinuousTraceUpload, TraceConsentSnapshot};+mod upload;use upload::perform_agent_trace_upload;pub use upload::{upload_trace_file, TraceUploadOptions};@@ -287,20 +290,35 @@ impl TraceUploadSignature {// Session header + contextfn read_trace_session_header(path: &Path) -> Option<pa_types::session::SessionHeader> {+ read_trace_session_header_checked(path).ok().flatten()+}++fn read_trace_session_header_checked(+ path: &Path,+) -> std::io::Result<Option<pa_types::session::SessionHeader>> {use std::io::BufRead;- let file = std::fs::File::open(path).ok()?;+ let file = std::fs::File::open(path)?;let mut first_line = String::new();- std::io::BufReader::new(file)+ // Corrupt headers must not allocate an entire unbounded transcript line.+ if let Err(error) = std::io::Read::take(std::io::BufReader::new(file), 256 * 1024 + 1).read_line(&mut first_line)- .ok()?;- if first_line.trim().is_empty() {- return None;+ {+ return if error.kind() == std::io::ErrorKind::InvalidData {+ Ok(None)+ } else {+ Err(error)+ };+ }+ if first_line.len() > 256 * 1024 || first_line.trim().is_empty() {+ return Ok(None);}- let value: Value = serde_json::from_str(first_line.trim()).ok()?;+ let Ok(value) = serde_json::from_str::<Value>(first_line.trim()) else {+ return Ok(None);+ };if !is_trace_session_header(&value) {- return None;+ return Ok(None);}- serde_json::from_value(value).ok()+ Ok(serde_json::from_value(value).ok())}fn is_trace_session_header(value: &Value) -> bool {@@ -510,8 +528,17 @@ fn read_agent_trace_outbox_entry(agent_dir: &Path,session_file: &Path,) -> Option<TraceUploadSignature> {+ use std::io::Read;let entry_path = agent_trace_outbox_entry_path(agent_dir, session_file);- let raw = std::fs::read_to_string(entry_path).ok()?;+ let mut raw = String::new();+ std::fs::File::open(entry_path)+ .ok()?+ .take(64 * 1024 + 1)+ .read_to_string(&mut raw)+ .ok()?;+ if raw.len() > 64 * 1024 {+ return None;+ }let (recorded_file, uploaded) = parse_outbox_entry(&raw)?;if recorded_file != session_file.to_string_lossy() {return None;@@ -523,6 +550,25 @@ fn signature_equals(recorded: Option<TraceUploadSignature>, current: TraceUploadrecorded.is_some_and(|recorded| recorded == current)}+// Separate short mutation leases from delivery leases (which cover network+// waits). Persist callers only try-lock; cursor/prune work stays in background.+fn outbox_mutation_lock(entry: &Path) -> std::io::Result<std::fs::File> {+ let lock = entry.with_extension("mutation-lock");+ let mut options = std::fs::OpenOptions::new();+ options.create(true).read(true).write(true).truncate(false);+ crate::platform::perms::set_private_mode(&mut options);+ match options.open(&lock) {+ Ok(file) => Ok(file),+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {+ if let Some(parent) = lock.parent() {+ std::fs::create_dir_all(parent)?;+ }+ options.open(lock)+ }+ Err(error) => Err(error),+ }+}+fn record_agent_trace_outbox_upload(agent_dir: &Path,session_file: &Path,@@ -530,6 +576,8 @@ fn record_agent_trace_outbox_upload() -> std::io::Result<()> {std::fs::create_dir_all(agent_trace_outbox_dir(agent_dir))?;let entry_path = agent_trace_outbox_entry_path(agent_dir, session_file);+ let mutation = outbox_mutation_lock(&entry_path)?;+ crate::platform::lock_exclusive(&mutation)?;let temp = entry_path.with_extension(format!("{}.{}.tmp",std::process::id(),diff --git a/crates/pa-core/src/agent_traces/continuous.rs b/crates/pa-core/src/agent_traces/continuous.rsnew file mode 100644index 0000000000..4f0723fa6e--- /dev/null+++ b/crates/pa-core/src/agent_traces/continuous.rs@@ -0,0 +1,1057 @@+//! Consent-gated automatic delivery. The detached process-wide runtime owns all+//! reads, recovery and delivery; hosts only record bounded intent and wake it.+use super::*;+use std::io::Read;+use std::sync::{Mutex, OnceLock, Weak};+use std::time::Duration;+use tokio::time::Instant;++const DEBOUNCE: Duration = Duration::from_secs(1);+const MIN_INTERVAL: Duration = Duration::from_secs(60);+const MAX_CONTROLLERS: usize = 256;++#[derive(Default, Debug)]+struct Schedule {+ due: Option<Instant>,+ last_start: Option<Instant>,+ not_before: Option<Instant>,+ generation: u64,+}++impl Schedule {+ fn persist(&mut self, now: Instant) {+ self.generation = self.generation.wrapping_add(1);+ self.due = Some(self.deadline(now));+ }++ fn deadline(&self, now: Instant) -> Instant {+ let mut due = now + DEBOUNCE;+ if let Some(start) = self.last_start {+ due = due.max(start + MIN_INTERVAL);+ }+ if let Some(not_before) = self.not_before {+ due = due.max(not_before);+ }+ due+ }++ fn start(&mut self, now: Instant) -> u64 {+ self.due = None;+ self.last_start = Some(now);+ self.generation+ }++ fn settle(&mut self, now: Instant, started_generation: u64, result: &TraceUploadResult) {+ let retry = matches!(+ result,+ TraceUploadResult::Disabled | TraceUploadResult::MissingCredentials+ ) || matches!(result, TraceUploadResult::Failed { status_code, .. }+ if status_code.is_none_or(|status| status == 429 || RETRIABLE_HTTP_STATUSES.contains(&status)));+ if let TraceUploadResult::Failed {+ retry_after_ms: Some(ms),+ ..+ } = result+ {+ self.not_before = Some(now + Duration::from_millis(*ms));+ }+ if retry || self.generation != started_generation {+ self.due = Some(self.deadline(now));+ }+ }+}++// The generation captures a snapshot so a poisoned metadata read fails closed+// at persist time; a changed settings generation only re-arms delivery, and+// parsing/reloading (the consent verdict) stays in the worker.+#[derive(Debug, Clone, PartialEq, Eq)]+struct ConsentGeneration(Vec<Result<Option<(u64, SystemTime)>, std::io::ErrorKind>>);++impl ConsentGeneration {+ fn read(cwd: &Path, agent_dir: &Path) -> Self {+ Self(+ [+ agent_dir.join("settings.json"),+ cwd.join(crate::settings::storage::CONFIG_DIR_NAME)+ .join("settings.json"),+ ]+ .iter()+ .map(|path| match std::fs::metadata(path) {+ Ok(m) => m+ .modified()+ .map(|mtime| Some((m.len(), mtime)))+ .map_err(|e| e.kind()),+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),+ Err(error) => Err(error.kind()),+ })+ .collect(),+ )+ }+}++/// Consent captured around the host's settings load. The generation must be+/// captured before parsing so a concurrent settings change fails closed.+#[derive(Clone)]+pub struct TraceConsentSnapshot {+ enabled: bool,+ generation: ConsentGeneration,+}++/// Session-owned installation. Dropping the last host reference cancels delivery;+/// no host shutdown awaits this worker or its retry timers.+pub struct ContinuousTraceUpload {+ cwd: PathBuf,+ agent_dir: Arc<PathBuf>,+ consent: Mutex<(bool, ConsentGeneration)>,+ // The path is passed by the writer: forks/rebindings cannot upload a stale path.+ pending: Mutex<Option<(PathBuf, Schedule)>>,+ wake: Arc<tokio::sync::Notify>,+ cancel: TraceUploadCancel,+ started: AtomicBool,+}++impl std::fmt::Debug for ContinuousTraceUpload {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("ContinuousTraceUpload")+ .finish_non_exhaustive()+ }+}++impl Drop for ContinuousTraceUpload {+ fn drop(&mut self) {+ self.cancel.cancel();+ self.wake.notify_one();+ }+}++impl ContinuousTraceUpload {+ /// Reuse this settings load for the host's other settings. Only two bounded+ /// metadata reads are added; parsing is the existing host configuration load.+ #[must_use]+ pub fn load_settings(+ cwd: &Path,+ agent_dir: &Path,+ ) -> (crate::settings::SettingsManager, TraceConsentSnapshot) {+ let generation = ConsentGeneration::read(cwd, agent_dir);+ let settings = crate::settings::SettingsManager::create(cwd, agent_dir);+ let consent = TraceConsentSnapshot {+ enabled: settings.errors().is_empty() && settings.get_agent_traces_enabled(),+ generation,+ };+ (settings, consent)+ }++ fn consent_for(&self, cwd: &Path) -> TraceConsentSnapshot {+ let Ok(consent) = self.consent.lock() else {+ return TraceConsentSnapshot {+ enabled: false,+ generation: ConsentGeneration::read(cwd, &self.agent_dir),+ };+ };+ TraceConsentSnapshot {+ enabled: cwd == self.cwd && consent.0,+ generation: if cwd == self.cwd {+ consent.1.clone()+ } else {+ ConsentGeneration::read(cwd, &self.agent_dir)+ },+ }+ }++ /// Retire this installation and bind its registry slot to a replacement.+ /// A cwd change starts with consent off until background verification.+ #[must_use]+ pub fn rebind(&self, cwd: &Path, path: &Path) -> Option<Arc<Self>> {+ let controller =+ Self::unregistered(cwd, &self.agent_dir, Some(path), self.consent_for(cwd));+ if service().replace(self, &controller) {+ Some(controller)+ } else {+ tracing::warn!("trace replacement rejected: source registration unavailable");+ None+ }+ }++ /// Fork within this installation's cwd, admitting a distinct installation.+ #[must_use]+ pub fn forked(&self, path: &Path) -> Option<Arc<Self>> {+ Self::install(+ &self.cwd,+ &self.agent_dir,+ Some(path),+ self.consent_for(&self.cwd),+ )+ }++ fn unregistered(+ cwd: &Path,+ agent_dir: &Path,+ session_file: Option<&Path>,+ consent: TraceConsentSnapshot,+ ) -> Arc<Self> {+ Arc::new(Self {+ cwd: cwd.to_path_buf(),+ agent_dir: Arc::new(agent_dir.to_path_buf()),+ consent: Mutex::new((consent.enabled, consent.generation)),+ pending: Mutex::new(session_file.map(|p| (p.to_path_buf(), Schedule::default()))),+ wake: Arc::new(tokio::sync::Notify::new()),+ cancel: TraceUploadCancel::new(),+ started: AtomicBool::new(false),+ })+ }++ /// Install without scanning the outbox or waiting for networking. Consent+ /// is captured around the host's existing settings load.+ /// Returns `None` when the bounded registry cannot admit this installation.+ /// No persist hook may be attached to a rejected installation.+ #[must_use]+ pub fn install(+ cwd: &Path,+ agent_dir: &Path,+ session_file: Option<&Path>,+ consent: TraceConsentSnapshot,+ ) -> Option<Arc<Self>> {+ let controller = Self::unregistered(cwd, agent_dir, session_file, consent);+ let service = service();+ if service.register(&controller) {+ Some(controller)+ } else {+ tracing::warn!("trace installation rejected: registry unavailable or at capacity");+ None+ }+ }++ /// Called only after a successful transcript write. This performs no+ /// transcript read, settings parse, directory scan or network. Only the small+ /// pending record is serialized synchronously.+ pub fn persisted(&self, session_file: &Path) {+ if session_file.as_os_str().is_empty() || self.cancel.is_cancelled() {+ return;+ }+ let Ok(consent) = self.consent.lock() else {+ return;+ };+ if !consent.0 || consent.1 .0.iter().any(Result::is_err) {+ return;+ }+ drop(consent);+ let began = std::time::Instant::now();+ if let Err(error) = mark_pending(&self.agent_dir, session_file) {+ tracing::warn!(%error, "trace pending marker failed");+ }+ tracing::trace!(+ elapsed_us = began.elapsed().as_micros() as u64,+ "trace pending marker duration"+ );+ let Ok(mut pending) = self.pending.lock() else {+ return;+ };+ let (path, schedule) =+ pending.get_or_insert_with(|| (session_file.to_path_buf(), Schedule::default()));+ if path != session_file {+ *path = session_file.to_path_buf();+ *schedule = Schedule::default();+ }+ schedule.persist(Instant::now());+ drop(pending);+ self.wake.notify_one();+ }+}++fn mark_pending(agent_dir: &Path, session_file: &Path) -> std::io::Result<()> {+ let primary = agent_trace_outbox_entry_path(agent_dir, session_file);+ let mutation = outbox_mutation_lock(&primary)?;+ // Never wait behind a pruner or cursor writer on the host thread. A stable+ // fallback marker retains intent even if the primary is about to be pruned.+ let acquired = crate::platform::try_lock_exclusive(&mutation).is_ok();+ let entry = if acquired {+ primary+ } else {+ primary.with_extension("pending.json")+ };+ if entry.exists() {+ return Ok(());+ }+ std::fs::create_dir_all(agent_trace_outbox_dir(agent_dir))?;+ // Publish a complete record without replacing a racing successful cursor.+ let temp = entry.with_extension(format!(+ "{}.{}.tmp",+ std::process::id(),+ uuid::Uuid::new_v4()+ ));+ let result = (|| {+ use std::io::Write;+ let mut options = std::fs::OpenOptions::new();+ options.write(true).create_new(true);+ crate::platform::perms::set_private_mode(&mut options);+ let mut file = options.open(&temp)?;+ writeln!(+ file,+ "{}",+ json!({"sessionFile": session_file.to_string_lossy()})+ )?;+ match std::fs::hard_link(&temp, &entry) {+ Ok(()) => Ok(()),+ Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Ok(()),+ Err(error) => Err(error),+ }+ })();+ let _ = std::fs::remove_file(temp);+ result+}++fn prune_entry(entry: &Path, observed: &str, missing_session: Option<&Path>) {+ // Fallback markers are deliberately retained: another process can publish+ // intent there without waiting while the primary's short lease is held.+ if entry+ .file_name()+ .is_some_and(|name| name.to_string_lossy().ends_with(".pending.json"))+ {+ return;+ }+ let Ok(mutation) = outbox_mutation_lock(entry) else {+ return;+ };+ if crate::platform::try_lock_exclusive(&mutation).is_err() {+ return;+ }+ let mut current = String::new();+ let Ok(file) = std::fs::File::open(entry) else {+ return;+ };+ if file+ .take(64 * 1024 + 1)+ .read_to_string(&mut current)+ .is_err()+ || current != observed+ {+ return;+ }+ if let Some(session) = missing_session {+ match std::fs::metadata(session) {+ Ok(meta) if meta.is_file() => return,+ Ok(_) => {}+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}+ Err(_) => return,+ }+ }+ if let Err(error) = std::fs::remove_file(entry) {+ tracing::debug!(%error, "trace stale marker pruning failed");+ }+}++fn delivery_lease(agent_dir: &Path, path: &Path) -> Option<std::fs::File> {+ let lock = agent_trace_outbox_entry_path(agent_dir, path).with_extension("lock");+ std::fs::create_dir_all(agent_trace_outbox_dir(agent_dir)).ok()?;+ let mut options = std::fs::OpenOptions::new();+ options.create(true).read(true).write(true).truncate(false);+ crate::platform::perms::set_private_mode(&mut options);+ let file = options.open(lock).ok()?;+ crate::platform::try_lock_exclusive(&file).ok()?;+ Some(file)+}++#[derive(Clone)]+struct Registration {+ controller: Weak<ContinuousTraceUpload>,+ agent_dir: Arc<PathBuf>,+ retired_predecessor: bool,+}++impl Registration {+ fn of(controller: &Arc<ContinuousTraceUpload>) -> Self {+ Self {+ controller: Arc::downgrade(controller),+ agent_dir: controller.agent_dir.clone(),+ retired_predecessor: false,+ }+ }+}++struct Service {+ controllers: Mutex<Vec<Registration>>,+ wake: tokio::sync::Notify,+}++impl Service {+ fn register(&self, controller: &Arc<ContinuousTraceUpload>) -> bool {+ let Ok(mut registrations) = self.controllers.lock() else {+ return false;+ };+ // Only the background service acknowledges dead descriptors, preserving+ // retirements between ticks. Rejection never returns a usable controller.+ if registrations.len() >= MAX_CONTROLLERS {+ return false;+ }+ registrations.push(Registration::of(controller));+ self.wake.notify_one();+ true+ }++ fn replace(+ &self,+ source: &ContinuousTraceUpload,+ controller: &Arc<ContinuousTraceUpload>,+ ) -> bool {+ let Ok(mut registrations) = self.controllers.lock() else {+ return false;+ };+ let Some(slot) = registrations+ .iter_mut()+ .find(|registration| std::ptr::eq(registration.controller.as_ptr(), source))+ else {+ return false;+ };+ // Reuse exactly one slot. The flag retains retirement even when neither+ // predecessor nor replacement survives until the next service tick.+ *slot = Registration::of(controller);+ slot.retired_predecessor = true;+ source.cancel.cancel();+ source.wake.notify_one();+ self.wake.notify_one();+ true+ }+}++fn service() -> &'static Service {+ static SERVICE: OnceLock<Service> = OnceLock::new();+ SERVICE.get_or_init(|| {+ let service = Service {+ controllers: Mutex::new(Vec::new()),+ wake: tokio::sync::Notify::new(),+ };+ // Detached OS threads do not extend process lifetime. The host's Tokio+ // blocking pool is never used by uploads or outbox recovery.+ if let Err(error) = std::thread::Builder::new()+ .name("trace-upload".into())+ .spawn(|| {+ let runtime = match tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ {+ Ok(runtime) => runtime,+ Err(error) => {+ tracing::warn!(%error, "trace runtime failed");+ return;+ }+ };+ runtime.block_on(run_service());+ })+ {+ tracing::warn!(%error, "trace thread failed");+ }+ service+ })+}++// Recovery phases: 0 pending, 1 running/cooling down, 2 completed.+struct RecoveryRun {+ cancel: TraceUploadCancel,+ state: Arc<std::sync::atomic::AtomicU8>,+ replay_needed: Arc<AtomicBool>,+ not_before: Arc<Mutex<Option<Instant>>>,+ // Weak identities pin allocations, so retirement detection cannot reuse addresses.+ hosts: Vec<Weak<ContinuousTraceUpload>>,+}++impl RecoveryRun {+ fn new(hosts: Vec<Weak<ContinuousTraceUpload>>) -> Self {+ Self {+ cancel: TraceUploadCancel::new(),+ state: Arc::new(std::sync::atomic::AtomicU8::new(0)),+ replay_needed: Arc::new(AtomicBool::new(false)),+ not_before: Arc::new(Mutex::new(None)),+ hosts,+ }+ }+}++type RecoveryRuns = std::collections::HashMap<PathBuf, RecoveryRun>;++fn retain_live_recoveries(+ recovered: &mut RecoveryRuns,+ registrations: &[Registration],+) -> Vec<Weak<ContinuousTraceUpload>> {+ let mut live: std::collections::HashMap<PathBuf, Vec<Weak<ContinuousTraceUpload>>> =+ std::collections::HashMap::new();+ let mut retired_directories = std::collections::HashSet::new();+ let mut retired = Vec::new();+ for registration in registrations {+ if registration.retired_predecessor {+ retired_directories.insert(registration.agent_dir.as_ref().clone());+ }+ if let Some(controller) = registration.controller.upgrade() {+ live.entry(controller.agent_dir.as_ref().clone())+ .or_default()+ .push(registration.controller.clone());+ } else {+ retired_directories.insert(registration.agent_dir.as_ref().clone());+ retired.push(registration.controller.clone());+ }+ }+ recovered.retain(|directory, run| {+ let Some(hosts) = live.get(directory) else {+ run.cancel.cancel();+ return false;+ };+ if retired_directories.contains(directory)+ || run+ .hosts+ .iter()+ .any(|old| !hosts.iter().any(|new| old.ptr_eq(new)))+ {+ run.replay_needed.store(true, Ordering::Release);+ }+ run.hosts.clone_from(hosts);+ true+ });+ retired+}++fn acknowledge_retired_registrations(+ registrations: &mut Vec<Registration>,+ retired: &[Weak<ContinuousTraceUpload>],+) {+ // The retained Weak handles pin allocations until acknowledgement finishes.+ // Hosts retiring after the snapshot remain registered for the next tick.+ let identities: std::collections::HashSet<_> =+ retired.iter().map(|weak| weak.as_ptr() as usize).collect();+ registrations+ .retain(|registration| !identities.contains(&(registration.controller.as_ptr() as usize)));+}++fn acknowledge_replacements(registrations: &mut [Registration], snapshot: &[Registration]) {+ let identities: std::collections::HashSet<_> = snapshot+ .iter()+ .filter(|registration| registration.retired_predecessor)+ .map(|registration| registration.controller.as_ptr() as usize)+ .collect();+ // Snapshot Weak handles pin identity; a later replacement keeps its flag.+ for registration in registrations {+ if identities.contains(&(registration.controller.as_ptr() as usize)) {+ registration.retired_predecessor = false;+ }+ }+}++fn rearm_retired_recovery(run: &RecoveryRun) {+ if run.replay_needed.load(Ordering::Acquire)+ && run+ .state+ .compare_exchange(2, 1, Ordering::AcqRel, Ordering::Acquire)+ .is_ok()+ {+ let state = run.state.clone();+ let cancel = run.cancel.clone();+ let not_before = *run.not_before.lock().unwrap();+ tokio::spawn(async move {+ settle_recovery_run(&state, false, not_before, &cancel).await;+ });+ }+}++async fn run_service() {+ let service = service();+ let permits = Arc::new(tokio::sync::Semaphore::new(4));+ // Recovery belongs to all live hosts sharing this directory, not its first host.+ let mut recovered = RecoveryRuns::new();+ loop {+ let registrations = service.controllers.lock().unwrap().clone();+ let retired = retain_live_recoveries(&mut recovered, ®istrations);+ {+ let mut current = service.controllers.lock().unwrap();+ acknowledge_retired_registrations(&mut current, &retired);+ acknowledge_replacements(&mut current, ®istrations);+ }+ for registration in registrations {+ let weak = registration.controller;+ if let Some(controller) = weak.upgrade() {+ if controller.consent.lock().unwrap().0+ && (recovered.contains_key(controller.agent_dir.as_ref())+ || recovered.len() < MAX_CONTROLLERS)+ {+ let run = recovered+ .entry(controller.agent_dir.as_ref().clone())+ .or_insert_with(|| RecoveryRun::new(vec![weak.clone()]));+ rearm_retired_recovery(run);+ if run+ .state+ .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire)+ .is_ok()+ {+ let cwd = controller.cwd.clone();+ let agent_dir = controller.agent_dir.as_ref().clone();+ // Clear only on admission: retirement during completion must survive.+ run.replay_needed.store(false, Ordering::Release);+ let cancel = run.cancel.clone();+ let state = run.state.clone();+ let replay_needed = run.replay_needed.clone();+ let saved_deadline = run.not_before.clone();+ let permits = permits.clone();+ tokio::spawn(async move {+ let (complete, not_before) = recover_with_backoff(+ cwd,+ agent_dir,+ permits,+ Arc::new(ReqwestTraceHttp),+ None,+ cancel.clone(),+ None,+ )+ .await;+ *saved_deadline.lock().unwrap() = not_before;+ let complete = complete && !replay_needed.load(Ordering::Acquire);+ settle_recovery_run(&state, complete, not_before, &cancel).await;+ });+ }+ }+ if !controller.started.swap(true, Ordering::AcqRel) {+ tokio::spawn(run_controller(+ weak,+ permits.clone(),+ Arc::new(ReqwestTraceHttp),+ None,+ ));+ }+ }+ }+ tokio::select! { () = service.wake.notified() => {}, () = tokio::time::sleep(DEBOUNCE) => {} }+ }+}++async fn settle_recovery_run(+ state: &std::sync::atomic::AtomicU8,+ complete: bool,+ not_before: Option<Instant>,+ cancel: &TraceUploadCancel,+) {+ if !complete {+ // Mixed-consent directories may need another background sweep. Keep its+ // state running during cooldown; the service rechecks consent before rearm.+ let due = (Instant::now() + MIN_INTERVAL).max(not_before.unwrap_or_else(Instant::now));+ tokio::select! {+ () = tokio::time::sleep_until(due) => {},+ () = cancel.wait() => {},+ }+ }+ state.store(if complete { 2 } else { 0 }, Ordering::Release);+}++async fn run_controller(+ weak: Weak<ContinuousTraceUpload>,+ permits: Arc<tokio::sync::Semaphore>,+ http: Arc<dyn TraceHttp>,+ base_url: Option<String>,+) {+ if let Some(controller) = weak.upgrade() {+ let path = controller+ .pending+ .lock()+ .unwrap()+ .as_ref()+ .map(|(p, _)| p.clone());+ if let Some(path) = path {+ let entry = agent_trace_outbox_entry_path(&controller.agent_dir, &path);+ let pending_on_disk = entry.is_file() || entry.with_extension("pending.json").is_file();+ if pending_on_disk {+ let mut pending = controller.pending.lock().unwrap();+ if let Some((current, schedule)) = pending.as_mut() {+ if current == &path {+ schedule.persist(Instant::now());+ }+ }+ }+ }+ }+ let mut initialized = false;+ loop {+ let Some(controller) = weak.upgrade() else {+ return;+ };+ let generation = ConsentGeneration::read(&controller.cwd, &controller.agent_dir);+ if controller.consent.lock().unwrap().1 != generation || !initialized {+ let settings = crate::settings::SettingsManager::create(+ &controller.cwd,+ controller.agent_dir.as_ref(),+ );+ let unchanged =+ generation == ConsentGeneration::read(&controller.cwd, &controller.agent_dir);+ *controller.consent.lock().unwrap() = (+ unchanged && settings.errors().is_empty() && settings.get_agent_traces_enabled(),+ generation,+ );+ initialized = true;+ }+ let enabled = controller.consent.lock().unwrap().0;+ let due = controller+ .pending+ .lock()+ .unwrap()+ .as_ref()+ .and_then(|(_, s)| s.due);+ let wake = controller.wake.clone();+ let cancel = controller.cancel.clone();+ // Weak ownership during waits is essential: timers cannot keep a session alive.+ drop(controller);+ if !enabled || due.is_none_or(|due| due > Instant::now()) {+ // Overdue work must not spin or acquire capacity while consent is off.+ let wait = if enabled {+ due.map_or(DEBOUNCE, |due| {+ due.saturating_duration_since(Instant::now()).min(DEBOUNCE)+ })+ } else {+ DEBOUNCE+ };+ tokio::select! { () = wake.notified() => {}, () = tokio::time::sleep(wait) => {}, () = cancel.wait() => return }+ continue;+ }+ let permit = tokio::select! { p = permits.clone().acquire_owned() => p.unwrap(), () = cancel.wait() => return };+ let Some(controller) = weak.upgrade() else {+ return;+ };+ let path = controller+ .pending+ .lock()+ .unwrap()+ .as_ref()+ .map(|(path, _)| path.clone());+ let Some(path) = path else {+ continue;+ };+ let Some(_delivery_lease) = delivery_lease(&controller.agent_dir, &path) else {+ // A busy lease must not hold delivery capacity while waiting.+ drop(permit);+ drop(controller);+ tokio::select! { () = tokio::time::sleep(DEBOUNCE) => {}, () = cancel.wait() => return }+ continue;+ };+ let leased_path = path;+ let (path, generation) = {+ let mut pending = controller.pending.lock().unwrap();+ let Some((path, schedule)) = pending.as_mut() else {+ continue;+ };+ if path != &leased_path || schedule.due.is_none_or(|due| due > Instant::now()) {+ continue;+ }+ (path.clone(), schedule.start(Instant::now()))+ };+ let cwd = controller.cwd.clone();+ let agent_dir = controller.agent_dir.clone();+ drop(controller);+ let result = deliver_with_consent(+ &TraceUploadOptions {+ session_file: Some(&path),+ cwd: &cwd,+ agent_dir: &agent_dir,+ require_enabled: true,+ reload_config: true,+ base_url: base_url.as_deref(),+ http: http.as_ref(),+ request_timeout_ms: DEFAULT_REQUEST_TIMEOUT_MS,+ cancel: Some(&cancel),+ on_upload_delay: None,+ },+ None,+ )+ .await;+ log_agent_trace_outcome(&agent_dir, Some(&path), &result);+ drop(permit);+ if let Some(controller) = weak.upgrade() {+ let mut pending = controller.pending.lock().unwrap();+ if let Some((current, schedule)) = pending.as_mut() {+ if current == &path {+ schedule.settle(Instant::now(), generation, &result);+ }+ }+ }+ }+}++async fn deliver_with_consent(+ options: &TraceUploadOptions<'_>,+ gate: Option<&TraceRequestGate>,+) -> TraceUploadResult {+ let mut initial = ConsentGeneration::read(options.cwd, options.agent_dir);+ let request_cancel = TraceUploadCancel::new();+ let guarded = TraceUploadOptions {+ cancel: Some(&request_cancel),+ on_upload_delay: options.on_upload_delay.clone(),+ ..*options+ };+ let request = perform_agent_trace_upload(&guarded, gate);+ tokio::pin!(request);+ loop {+ tokio::select! {+ result = &mut request => return result,+ () = async { if let Some(cancel) = options.cancel { cancel.wait().await; } else { std::future::pending::<()>().await; } } => {+ request_cancel.cancel();+ return request.await;+ }+ () = tokio::time::sleep(DEBOUNCE) => {+ let current = ConsentGeneration::read(options.cwd, options.agent_dir);+ if initial != current {+ let settings = crate::settings::SettingsManager::create(options.cwd, options.agent_dir);+ if settings.errors().is_empty() && settings.get_agent_traces_enabled()+ && current == ConsentGeneration::read(options.cwd, options.agent_dir) {+ initial = current;+ } else {+ request_cancel.cancel();+ let _ = request.await;+ return TraceUploadResult::Disabled;+ }+ }+ }+ }+ }+}++async fn recover_with_backoff(+ cwd: PathBuf,+ agent_dir: PathBuf,+ permits: Arc<tokio::sync::Semaphore>,+ http: Arc<dyn TraceHttp>,+ base_url: Option<String>,+ cancel: TraceUploadCancel,+ live_controllers: Option<Vec<Weak<ContinuousTraceUpload>>>,+) -> (bool, Option<Instant>) {+ recover_with_io(+ cwd,+ agent_dir,+ permits,+ http,+ base_url,+ cancel,+ live_controllers,+ |_, _| Ok(()),+ )+ .await+}++#[derive(Clone, Copy, PartialEq, Eq, Debug)]+enum RecoveryRead {+ Directory,+ Iteration,+ RecordMetadata,+ Record,+ SessionMetadata,+ Header,+}++// Keep fault injection at the I/O boundary so deterministic recovery tests can+// exercise transient failures without changing filesystem permissions or users.+#[allow(clippy::too_many_arguments)]+async fn recover_with_io(+ cwd: PathBuf,+ agent_dir: PathBuf,+ permits: Arc<tokio::sync::Semaphore>,+ http: Arc<dyn TraceHttp>,+ base_url: Option<String>,+ cancel: TraceUploadCancel,+ live_controllers: Option<Vec<Weak<ContinuousTraceUpload>>>,+ before_read: impl Fn(RecoveryRead, &Path) -> std::io::Result<()> + Send + Sync,+) -> (bool, Option<Instant>) {+ let settings = crate::settings::SettingsManager::create(&cwd, &agent_dir);+ if !settings.errors().is_empty() || !settings.get_agent_traces_enabled() {+ return (false, None);+ }+ // Workers share an agent directory: only one startup sweep may deliver it+ // at a time. OS locks release on crashes without stale-directory retries.+ let Some(_recovery_lease) = delivery_lease(&agent_dir, &agent_dir.join("catch-up")) else {+ return (false, None);+ };+ let outbox = agent_trace_outbox_dir(&agent_dir);+ let read_directory = async {+ before_read(RecoveryRead::Directory, &outbox)?;+ tokio::fs::read_dir(&outbox).await+ };+ let mut entries = match read_directory.await {+ Ok(entries) => entries,+ Err(error) => return (error.kind() == std::io::ErrorKind::NotFound, None),+ };+ let gate = TraceRequestGate::new();+ let mut complete = true;+ let mut not_before: Option<Instant> = None;+ loop {+ let next = async {+ before_read(RecoveryRead::Iteration, &outbox)?;+ entries.next_entry().await+ };+ let entry = match next.await {+ Ok(Some(entry)) => entry,+ Ok(None) => break,+ Err(error) => {+ // A transient directory-iteration failure must not let the+ // sweep complete: unexamined markers stay eligible and the+ // bounded cooldown re-arms another sweep.+ tracing::warn!(%error, "trace outbox iteration failed");+ complete = false;+ break;+ }+ };+ if cancel.is_cancelled() {+ return (false, None);+ }+ if entry.path().extension().is_none_or(|ext| ext != "json") {+ continue;+ }+ // Outbox records contain only a path/cursor; bound corrupt record reads.+ let record_metadata = async {+ before_read(RecoveryRead::RecordMetadata, &entry.path())?;+ entry.metadata().await+ };+ match record_metadata.await {+ Ok(metadata) if metadata.len() > 64 * 1024 => continue,+ Ok(_) => {}+ Err(_) => {+ complete = false;+ continue;+ }+ }+ let read_entry = async {+ use tokio::io::AsyncReadExt;+ before_read(RecoveryRead::Record, &entry.path())?;+ let file = tokio::fs::File::open(entry.path()).await?;+ let mut raw = String::new();+ file.take(64 * 1024 + 1).read_to_string(&mut raw).await?;+ Ok::<_, std::io::Error>(raw)+ };+ let raw = match read_entry.await {+ Ok(raw) => raw,+ Err(error) => {+ // Invalid UTF-8 is terminal data, not a transient filesystem fault.+ complete &= error.kind() == std::io::ErrorKind::InvalidData;+ continue;+ }+ };+ if raw.len() > 64 * 1024 {+ continue;+ }+ let Ok(value) = serde_json::from_str::<Value>(&raw) else {+ prune_entry(&entry.path(), &raw, None);+ continue;+ };+ if value.get("kind").is_some() {+ continue;+ }+ let Some(path) = value+ .get("sessionFile")+ .and_then(Value::as_str)+ .map(PathBuf::from)+ else {+ prune_entry(&entry.path(), &raw, None);+ continue;+ };+ // Fulfilled cursor entries owe no bytes even when their owner opted out.+ if TraceUploadSignature::of(&path).is_some_and(|signature| {+ read_agent_trace_outbox_entry(&agent_dir, &path) == Some(signature)+ }) {+ continue;+ }+ let registered = live_controllers.clone().unwrap_or_else(|| {+ service()+ .controllers+ .lock()+ .unwrap()+ .iter()+ .map(|registration| registration.controller.clone())+ .collect()+ });+ let live = registered.iter().filter_map(Weak::upgrade).any(|c| {+ c.pending+ .lock()+ .unwrap()+ .as_ref()+ .is_some_and(|(p, _)| p == &path)+ });+ if live {+ complete = false;+ continue;+ }+ let session_metadata = async {+ before_read(RecoveryRead::SessionMetadata, &path)?;+ tokio::fs::metadata(&path).await+ };+ match session_metadata.await {+ Ok(meta) if meta.is_file() => {}+ Ok(_) => {+ prune_entry(&entry.path(), &raw, Some(&path));+ continue;+ }+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {+ prune_entry(&entry.path(), &raw, Some(&path));+ continue;+ }+ // A transient metadata error must not let the sweep complete.+ Err(_) => {+ complete = false;+ continue;+ }+ }+ let Some(_delivery_lease) = delivery_lease(&agent_dir, &path) else {+ // Another process owns this delivery; a skipped marker must not+ // complete the sweep or it strands when that process fails.+ complete = false;+ continue;+ };+ let header = before_read(RecoveryRead::Header, &path)+ .and_then(|()| read_trace_session_header_checked(&path));+ let header = match header {+ Ok(Some(header)) => header,+ Ok(None) => continue,+ Err(_) => {+ complete = false;+ continue;+ }+ };+ let session_cwd = PathBuf::from(&header.cwd);+ let options = TraceUploadOptions {+ session_file: Some(&path),+ cwd: &session_cwd,+ agent_dir: &agent_dir,+ require_enabled: true,+ reload_config: true,+ base_url: base_url.as_deref(),+ http: http.as_ref(),+ request_timeout_ms: DEFAULT_REQUEST_TIMEOUT_MS,+ cancel: Some(&cancel),+ on_upload_delay: None,+ };+ // Bound catch-up retries per entry; exhaustion leaves its durable marker.+ let mut schedule = Schedule::default();+ for attempt in 0..3 {+ let permit = tokio::select! { p = permits.clone().acquire_owned() => p.unwrap(), () = cancel.wait() => return (false, None) };+ let generation = schedule.start(Instant::now());+ let result = deliver_with_consent(&options, Some(&gate)).await;+ log_agent_trace_outcome(&agent_dir, Some(&path), &result);+ drop(permit);+ if matches!(+ result,+ TraceUploadResult::Disabled | TraceUploadResult::MissingCredentials+ ) {+ // A revoked project must not block other opted-in projects, and+ // missing credentials may arrive later. Retain the marker and+ // rearm the shared sweep at a bounded rate.+ complete = false;+ break;+ }+ schedule.settle(Instant::now(), generation, &result);+ if let Some(deadline) = schedule.not_before {+ not_before = Some(not_before.map_or(deadline, |old| old.max(deadline)));+ }+ let Some(due) = schedule.due else {+ break;+ };+ if attempt == 2 {+ // Exhausted per-entry retries with work still due: the durable+ // marker is retained, so the sweep must rearm, not complete.+ complete = false;+ break;+ }+ tokio::select! { () = tokio::time::sleep_until(due) => {}, () = cancel.wait() => return (false, None) }+ }+ }+ (complete && !cancel.is_cancelled(), not_before)+}++#[cfg(test)]+mod tests;diff --git a/crates/pa-core/src/agent_traces/continuous/tests.rs b/crates/pa-core/src/agent_traces/continuous/tests.rsnew file mode 100644index 0000000000..9260ddbd18--- /dev/null+++ b/crates/pa-core/src/agent_traces/continuous/tests.rs@@ -0,0 +1,1536 @@+use super::*;+use crate::agent_traces::tests::{+ clear_trace_credentials, env_lock, response, Fixture, ScriptedTraceHttp,+};+use std::io::Write;++async fn recover(+ cwd: PathBuf,+ agent_dir: PathBuf,+ permits: Arc<tokio::sync::Semaphore>,+ http: Arc<dyn TraceHttp>,+ base_url: Option<String>,+ cancel: TraceUploadCancel,+) -> bool {+ recover_with_backoff(cwd, agent_dir, permits, http, base_url, cancel, None)+ .await+ .0+}++fn controller(fixture: &Fixture, path: &Path, enabled: bool) -> Arc<ContinuousTraceUpload> {+ Arc::new(ContinuousTraceUpload {+ cwd: fixture.cwd.clone(),+ agent_dir: Arc::new(fixture.agent_dir.clone()),+ consent: Mutex::new((+ enabled,+ ConsentGeneration::read(&fixture.cwd, &fixture.agent_dir),+ )),+ pending: Mutex::new(Some((path.to_path_buf(), Schedule::default()))),+ wake: Arc::new(tokio::sync::Notify::new()),+ cancel: TraceUploadCancel::new(),+ started: AtomicBool::new(false),+ })+}++#[test]+fn debounce_throttle_and_writes_in_flight_coalesce() {+ let now = Instant::now();+ let mut s = Schedule::default();+ s.persist(now);+ s.persist(now + Duration::from_millis(500));+ assert_eq!(s.due, Some(now + Duration::from_millis(1500)));+ let generation = s.start(now + Duration::from_millis(1500));+ s.persist(now + Duration::from_secs(2));+ s.settle(+ now + Duration::from_secs(3),+ generation,+ &TraceUploadResult::Unchanged,+ );+ assert_eq!(s.due, Some(now + Duration::from_millis(61500)));+ let generation = s.start(s.due.unwrap());+ s.settle(+ now + Duration::from_secs(62),+ generation,+ &TraceUploadResult::Unchanged,+ );+ assert_eq!(s.due, None);+}++#[test]+fn rate_limit_reschedule_honors_retry_after_and_terminal_failure_stops() {+ let now = Instant::now();+ let mut s = Schedule::default();+ let generation = s.start(now);+ s.settle(+ now + Duration::from_secs(1),+ generation,+ &TraceUploadResult::Failed {+ status_code: Some(429),+ message: "synthetic".into(),+ retry_after_ms: Some(120_000),+ },+ );+ assert_eq!(s.due, Some(now + Duration::from_secs(121)));+ let generation = s.start(s.due.unwrap());+ s.settle(+ now + Duration::from_secs(122),+ generation,+ &TraceUploadResult::Failed {+ status_code: Some(403),+ message: "synthetic".into(),+ retry_after_ms: None,+ },+ );+ assert_eq!(s.due, None);+}++#[test]+fn consent_off_never_registers_or_schedules_retroactively() {+ let fixture = Fixture::new();+ let path = fixture.write_session("off.jsonl", "off");+ let c = controller(&fixture, &path, false);+ c.persisted(&path);+ *c.consent.lock().unwrap() = (+ true,+ ConsentGeneration::read(&fixture.cwd, &fixture.agent_dir),+ );+ assert_eq!(c.pending.lock().unwrap().as_ref().unwrap().1.due, None);+ assert!(!agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());+}++#[tokio::test(start_paused = true)]+async fn external_consent_change_fails_closed_until_background_reload() {+ let fixture = Fixture::new();+ let path = fixture.write_session("changed.jsonl", "changed");+ let c = controller(&fixture, &path, true);+ std::fs::write(+ fixture.agent_dir.join("settings.json"),+ r#"{"agentTraces":{"enabled":false}}"#,+ )+ .unwrap();+ c.persisted(&path);+ // The durable intent survives the external change; only the worker's+ // consent re-read decides whether it is ever delivered.+ assert!(c.pending.lock().unwrap().as_ref().unwrap().1.due.is_some());+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: Arc::new(tokio::sync::Notify::new()),+ });+ let task = tokio::spawn(run_controller(+ Arc::downgrade(&c),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ ));+ tokio::time::advance(MIN_INTERVAL * 2).await;+ tokio::task::yield_now().await;+ assert!(+ observed.try_recv().is_err(),+ "a disabled project must never deliver its retained intent"+ );+ drop(c);+ task.await.unwrap();+}++#[test]+fn marker_is_complete_and_preserves_a_success_cursor() {+ let fixture = Fixture::new();+ let path = fixture.write_session("marker.jsonl", "marker");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let raw =+ std::fs::read_to_string(agent_trace_outbox_entry_path(&fixture.agent_dir, &path)).unwrap();+ assert_eq!(+ parse_outbox_entry(&raw),+ Some((path.to_string_lossy().into_owned(), None))+ );+ let signature = TraceUploadSignature::of(&path).unwrap();+ record_agent_trace_outbox_upload(&fixture.agent_dir, &path, signature).unwrap();+ mark_pending(&fixture.agent_dir, &path).unwrap();+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ Some(signature)+ );+}++#[test]+fn competing_delivery_is_nonblocking_and_crash_recoverable() {+ let fixture = Fixture::new();+ let path = fixture.write_session("lease.jsonl", "lease");+ let lease = delivery_lease(&fixture.agent_dir, &path).unwrap();+ assert!(delivery_lease(&fixture.agent_dir, &path).is_none());+ drop(lease);+ assert!(delivery_lease(&fixture.agent_dir, &path).is_some());+}++#[tokio::test(start_paused = true)]+async fn controller_drop_cancels_timer_without_waiting_for_delivery() {+ let fixture = Fixture::new();+ let path = fixture.write_session("drop.jsonl", "drop");+ let c = controller(&fixture, &path, true);+ c.persisted(&path);+ let cancel = c.cancel.clone();+ let http = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));+ let task = tokio::spawn(run_controller(+ Arc::downgrade(&c),+ Arc::new(tokio::sync::Semaphore::new(1)),+ http,+ None,+ ));+ drop(c);+ assert!(cancel.is_cancelled());+ task.await.unwrap();+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).is_file());+}++// Informational default CI measurement; calibrated host benchmarks own latency gates.+#[test]+fn measure_synchronous_pending_marker_cost() {+ let fixture = Fixture::new();+ let mut cold = Vec::new();+ let mut warm = Vec::new();+ for index in 0..1000 {+ let path = fixture.session_dir.join(format!("synthetic-{index}.jsonl"));+ let c = controller(&fixture, &path, true);+ let start = std::time::Instant::now();+ c.persisted(&path);+ cold.push(start.elapsed().as_nanos());+ let start = std::time::Instant::now();+ c.persisted(&path);+ warm.push(start.elapsed().as_nanos());+ }+ cold.sort_unstable();+ warm.sort_unstable();+ let report = format!(+ "{}\n",+ json!({"samples":1000,"cold_marker_ns":{"p50":cold[500],"p95":cold[950],"p99":cold[990]},"warm_marker_ns":{"p50":warm[500],"p95":warm[950],"p99":warm[990]},"fsync":false,"consent_metadata_checks_included":false,"cached_consent_check_included":true})+ );+ // Explicit stdout retains this informational measurement in hosted CI logs+ // even when the test harness captures println output for passing tests.+ std::io::stdout().write_all(report.as_bytes()).unwrap();+}++fn enable_synthetic_fixture(fixture: &Fixture) {+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);+ settings.set_agent_traces_enabled(true).unwrap();+ std::fs::write(+ fixture.agent_dir.join("auth.json"),+ r#"{"prime-agent-traces":{"type":"api_key","key":"synthetic-only"}}"#,+ )+ .unwrap();+}++struct ObservedSink {+ requests: tokio::sync::mpsc::UnboundedSender<(+ String,+ tokio::sync::oneshot::Sender<TraceHttpResponse>,+ )>,+ cancelled: Arc<tokio::sync::Notify>,+}++impl TraceHttp for ObservedSink {+ fn put<'a>(+ &'a self,+ _url: &'a str,+ _headers: Vec<(String, String)>,+ body: String,+ _timeout_ms: u64,+ cancel: Option<&'a TraceUploadCancel>,+ ) -> std::pin::Pin<+ Box<+ dyn std::future::Future<Output = Result<TraceHttpResponse, TraceHttpError>> + Send + 'a,+ >,+ > {+ Box::pin(async move {+ let (reply, result) = tokio::sync::oneshot::channel();+ self.requests.send((body, reply)).unwrap();+ tokio::select! {+ answer = result => Ok(answer.unwrap()),+ () = async { if let Some(cancel) = cancel { cancel.wait().await; } else { std::future::pending::<()>().await; } } => {+ self.cancelled.notify_one();+ Err(TraceHttpError::Cancelled)+ }+ }+ })+ }+}++#[tokio::test(start_paused = true)]+async fn dropping_an_active_controller_cancels_a_hanging_sink() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("hanging.jsonl", "hanging");+ let c = controller(&fixture, &path, true);+ c.persisted(&path);+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let cancelled = Arc::new(tokio::sync::Notify::new());+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: cancelled.clone(),+ });+ let task = tokio::spawn(run_controller(+ Arc::downgrade(&c),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ ));+ let (_body, _reply) = observed.recv().await.unwrap();+ drop(c);+ cancelled.notified().await;+ task.await.unwrap();+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ None+ );+}++#[tokio::test(start_paused = true)]+async fn disabling_consent_cancels_an_inflight_request_without_shutting_down_the_host() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("revoked-inflight.jsonl", "revoked");+ let c = controller(&fixture, &path, true);+ c.persisted(&path);+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let cancelled = Arc::new(tokio::sync::Notify::new());+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: cancelled.clone(),+ });+ let task = tokio::spawn(run_controller(+ Arc::downgrade(&c),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ ));+ let (_body, _reply) = observed.recv().await.unwrap();+ let first_start = Instant::now();+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);+ settings.set_agent_traces_enabled(false).unwrap();+ cancelled.notified().await;+ assert!(!c.cancel.is_cancelled());+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ None+ );+ tokio::time::advance(MIN_INTERVAL * 2).await;+ tokio::task::yield_now().await;+ assert!(observed.try_recv().is_err(), "consent off must not deliver");+ settings.set_agent_traces_enabled(true).unwrap();+ let (_body, _reply) = tokio::time::timeout(MIN_INTERVAL * 2, observed.recv())+ .await+ .expect("restoring consent resumes without another persist")+ .unwrap();+ assert!(Instant::now() - first_start >= MIN_INTERVAL);+ drop(c);+ task.await.unwrap();+}++#[tokio::test(start_paused = true)]+async fn recovery_reschedules_rate_limits_without_holding_an_upload_permit() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("recovery-retry.jsonl", "retry");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: Arc::new(tokio::sync::Notify::new()),+ });+ let permits = Arc::new(tokio::sync::Semaphore::new(1));+ let task = tokio::spawn(recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ permits.clone(),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ ));+ let (_body, first) = observed.recv().await.unwrap();+ let started = Instant::now();+ let mut limited = response(429, "{}");+ limited.retry_after = Some("120".into());+ first.send(limited).unwrap();+ // Acquire readiness demonstrates the retry timer does not consume delivery capacity.+ let permit = permits.acquire().await.unwrap();+ drop(permit);+ let (_body, second) = observed.recv().await.unwrap();+ assert!(Instant::now() - started >= Duration::from_secs(120));+ second.send(response(200, "{}")).unwrap();+ assert!(task.await.unwrap());+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ TraceUploadSignature::of(&path)+ );+}++#[tokio::test(start_paused = true)]+async fn recovery_shutdown_cancels_a_hanging_request_and_preserves_intent() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("recovery-shutdown.jsonl", "shutdown");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let cancelled = Arc::new(tokio::sync::Notify::new());+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: cancelled.clone(),+ });+ let cancel = TraceUploadCancel::new();+ let task = tokio::spawn(recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ cancel.clone(),+ ));+ let (_body, _reply) = observed.recv().await.unwrap();+ cancel.cancel();+ cancelled.notified().await;+ assert!(!task.await.unwrap());+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ None+ );+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());+}++#[tokio::test(start_paused = true)]+async fn writes_during_upload_have_a_followup_and_do_not_advance_the_old_cursor() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("coalesce.jsonl", "coalesce");+ let initial_signature = TraceUploadSignature::of(&path).unwrap();+ let c = controller(&fixture, &path, true);+ c.persisted(&path);+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: Arc::new(tokio::sync::Notify::new()),+ });+ let task = tokio::spawn(run_controller(+ Arc::downgrade(&c),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ ));+ let (first_body, first_reply) = observed.recv().await.unwrap();+ let first_start = Instant::now();+ {+ use std::io::Write;+ let mut file = std::fs::OpenOptions::new()+ .append(true)+ .open(&path)+ .unwrap();+ writeln!(+ file,+ "{}",+ json!({"type":"custom","id":"synthetic-appended"})+ )+ .unwrap();+ }+ c.persisted(&path);+ first_reply.send(response(200, "{}")).unwrap();+ let (second_body, _second_reply) = observed.recv().await.unwrap();+ assert!(Instant::now() - first_start >= MIN_INTERVAL);+ assert!(!first_body.contains("synthetic-appended"));+ assert!(second_body.contains("synthetic-appended"));+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ Some(initial_signature)+ );+ drop(c);+ task.await.unwrap();+}++#[tokio::test(start_paused = true)]+async fn opted_out_recovery_does_not_create_an_outbox() {+ let fixture = Fixture::new();+ let sink = Arc::new(ScriptedTraceHttp::new(vec![]));+ recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await;+ assert!(!agent_trace_outbox_dir(&fixture.agent_dir).exists());+}++#[tokio::test(start_paused = true)]+async fn startup_recovery_uploads_pending_and_prunes_missing_but_preserves_unknown_kinds() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("recovery.jsonl", "recovery");+ let missing = fixture.session_dir.join("missing.jsonl");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ mark_pending(&fixture.agent_dir, &missing).unwrap();+ let unknown = agent_trace_outbox_dir(&fixture.agent_dir).join("unknown.json");+ std::fs::write(+ &unknown,+ json!({"sessionFile":path,"kind":"semantic-edges"}).to_string(),+ )+ .unwrap();+ let malformed = agent_trace_outbox_dir(&fixture.agent_dir).join("malformed.json");+ std::fs::write(&malformed, "{").unwrap();+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));+ recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await;+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ TraceUploadSignature::of(&path)+ );+ assert!(!agent_trace_outbox_entry_path(&fixture.agent_dir, &missing).exists());+ assert!(!malformed.exists());+ assert!(unknown.exists());+}++#[test]+fn persist_racing_a_pruner_retains_a_complete_nonblocking_fallback_marker() {+ let fixture = Fixture::new();+ let path = fixture.write_session("prune-race.jsonl", "race");+ let entry = agent_trace_outbox_entry_path(&fixture.agent_dir, &path);+ std::fs::create_dir_all(entry.parent().unwrap()).unwrap();+ std::fs::write(&entry, "{").unwrap();+ let pruner = outbox_mutation_lock(&entry).unwrap();+ pruner.lock().unwrap();+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let fallback = entry.with_extension("pending.json");+ let raw = std::fs::read_to_string(&fallback).unwrap();+ assert_eq!(+ parse_outbox_entry(&raw),+ Some((path.to_string_lossy().into_owned(), None))+ );+ drop(pruner);+ prune_entry(&entry, "{", None);+ assert!(!entry.exists());+ assert!(fallback.exists());+ prune_entry(&fallback, &raw, None);+ assert!(fallback.exists());+}++#[test]+fn stale_prune_observations_never_remove_a_new_cursor_or_recreated_transcript() {+ let fixture = Fixture::new();+ let path = fixture.write_session("stale-prune.jsonl", "stale");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let entry = agent_trace_outbox_entry_path(&fixture.agent_dir, &path);+ let raw = std::fs::read_to_string(&entry).unwrap();+ let signature = TraceUploadSignature::of(&path).unwrap();+ record_agent_trace_outbox_upload(&fixture.agent_dir, &path, signature).unwrap();+ prune_entry(&entry, &raw, Some(&path));+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ Some(signature)+ );+ let raw = std::fs::read_to_string(&entry).unwrap();+ prune_entry(&entry, &raw, Some(&path));+ assert!(entry.exists());+}++#[tokio::test(start_paused = true)]+async fn fallback_only_restart_schedules_the_live_controller_without_a_fresh_write() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("fallback-only.jsonl", "fallback-only");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let primary = agent_trace_outbox_entry_path(&fixture.agent_dir, &path);+ let fallback = primary.with_extension("pending.json");+ std::fs::rename(&primary, &fallback).unwrap();+ let c = controller(&fixture, &path, true);+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: Arc::new(tokio::sync::Notify::new()),+ });+ let task = tokio::spawn(run_controller(+ Arc::downgrade(&c),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ ));+ let (body, _reply) = observed.recv().await.unwrap();+ assert!(body.contains("fallback-only"));+ drop(c);+ task.await.unwrap();+ assert!(fallback.exists());+}++#[test]+fn recovery_registry_releases_inactive_directory_capacity_and_preserves_other_hosts() {+ let fixture = Fixture::new();+ let c = controller(&fixture, &fixture.session_dir.join("registry.jsonl"), true);+ let another_host = controller(&fixture, &fixture.session_dir.join("other.jsonl"), true);+ let dead_cancel = TraceUploadCancel::new();+ let live_cancel = TraceUploadCancel::new();+ let mut recovered = RecoveryRuns::new();+ for n in 0..MAX_CONTROLLERS - 1 {+ recovered.insert(+ PathBuf::from(format!("synthetic-inactive-{n}")),+ RecoveryRun {+ cancel: dead_cancel.clone(),+ ..RecoveryRun::new(vec![])+ },+ );+ }+ recovered.insert(+ fixture.agent_dir,+ RecoveryRun {+ cancel: live_cancel.clone(),+ ..RecoveryRun::new(vec![Arc::downgrade(&c), Arc::downgrade(&another_host)])+ },+ );+ let registration = Registration::of(&c);+ drop(c);+ let _ = retain_live_recoveries(+ &mut recovered,+ &[registration, Registration::of(&another_host)],+ );+ assert!(dead_cancel.is_cancelled());+ assert!(!live_cancel.is_cancelled());+ assert_eq!(recovered.len(), 1);+ drop(another_host);+ let _ = retain_live_recoveries(&mut recovered, &[]);+ assert!(live_cancel.is_cancelled());+ assert!(recovered.is_empty());+}++#[tokio::test(start_paused = true)]+async fn revoked_recovery_retains_intent_and_resumes_after_consent_restoration() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("revoked-recovery.jsonl", "revoked-recovery");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let cancelled = Arc::new(tokio::sync::Notify::new());+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: cancelled.clone(),+ });+ let permits = Arc::new(tokio::sync::Semaphore::new(1));+ let task = tokio::spawn(recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ permits.clone(),+ sink.clone(),+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ ));+ let (_body, _reply) = observed.recv().await.unwrap();+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);+ settings.set_agent_traces_enabled(false).unwrap();+ cancelled.notified().await;+ assert!(!tokio::time::timeout(DEBOUNCE * 2, task)+ .await+ .unwrap()+ .unwrap());+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());+ assert!(delivery_lease(&fixture.agent_dir, &path).is_some());+ assert_eq!(permits.available_permits(), 1);+ assert!(+ !recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ permits.clone(),+ sink.clone(),+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await+ );+ assert!(observed.try_recv().is_err());+ settings.set_agent_traces_enabled(true).unwrap();+ let task = tokio::spawn(recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ permits,+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ ));+ let (_body, reply) = tokio::time::timeout(MIN_INTERVAL, observed.recv())+ .await+ .unwrap()+ .unwrap();+ reply.send(response(200, "{}")).unwrap();+ assert!(task.await.unwrap());+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ TraceUploadSignature::of(&path)+ );+}++#[tokio::test(start_paused = true)]+async fn incomplete_recovery_cooldown_is_bounded_and_shutdown_cancellable() {+ let state = Arc::new(std::sync::atomic::AtomicU8::new(1));+ let cancel = TraceUploadCancel::new();+ let task_state = state.clone();+ let task_cancel = cancel.clone();+ let task = tokio::spawn(async move {+ settle_recovery_run(&task_state, false, None, &task_cancel).await;+ });+ tokio::task::yield_now().await;+ tokio::time::advance(MIN_INTERVAL.checked_sub(DEBOUNCE).unwrap()).await;+ assert_eq!(state.load(Ordering::Acquire), 1);+ tokio::time::advance(DEBOUNCE).await;+ task.await.unwrap();+ assert_eq!(state.load(Ordering::Acquire), 0);+ state.store(1, Ordering::Release);+ let task_state = state.clone();+ let task_cancel = cancel.clone();+ let task = tokio::spawn(async move {+ settle_recovery_run(&task_state, false, None, &task_cancel).await;+ });+ tokio::task::yield_now().await;+ let began = Instant::now();+ cancel.cancel();+ task.await.unwrap();+ assert_eq!(Instant::now(), began);+ assert_eq!(state.load(Ordering::Acquire), 0);+}++fn mixed_consent_fixture() -> (Fixture, PathBuf, PathBuf, PathBuf) {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let first = fixture.write_session("disabled-project.jsonl", "disabled-project");+ let second = fixture.write_session("enabled-project.jsonl", "enabled-project");+ let second_cwd = fixture.agent_dir.join("other-project");+ std::fs::create_dir_all(&second_cwd).unwrap();+ for (path, cwd) in [(&first, &fixture.cwd), (&second, &second_cwd)] {+ let raw = std::fs::read_to_string(path).unwrap();+ let (header, tail) = raw.split_once('\n').unwrap();+ let mut header: Value = serde_json::from_str(header).unwrap();+ header["cwd"] = json!(cwd);+ std::fs::write(path, format!("{header}\n{tail}")).unwrap();+ mark_pending(&fixture.agent_dir, path).unwrap();+ }+ let config = fixture.cwd.join(crate::settings::storage::CONFIG_DIR_NAME);+ std::fs::create_dir_all(&config).unwrap();+ std::fs::write(+ config.join("settings.json"),+ r#"{"agentTraces":{"enabled":false}}"#,+ )+ .unwrap();+ (fixture, first, second, second_cwd)+}++#[tokio::test(start_paused = true)]+async fn disabled_project_recovery_does_not_block_other_opted_in_projects() {+ let (fixture, first, second, second_cwd) = mixed_consent_fixture();+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));+ assert!(+ !recover(+ second_cwd,+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await+ );+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &first),+ None+ );+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &first).exists());+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &second),+ TraceUploadSignature::of(&second)+ );+}++#[tokio::test(start_paused = true)]+async fn disabled_uploaded_cursor_does_not_rearm_recovery() {+ let (fixture, first, second, second_cwd) = mixed_consent_fixture();+ record_agent_trace_outbox_upload(+ &fixture.agent_dir,+ &first,+ TraceUploadSignature::of(&first).unwrap(),+ )+ .unwrap();+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));+ assert!(+ recover(+ second_cwd,+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await+ );+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &second),+ TraceUploadSignature::of(&second)+ );+}++#[tokio::test(start_paused = true)]+async fn mixed_consent_rescan_preserves_exhausted_rate_limit_deadline() {+ let (fixture, _first, _second, second_cwd) = mixed_consent_fixture();+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: Arc::new(tokio::sync::Notify::new()),+ });+ let task = tokio::spawn(recover_with_backoff(+ second_cwd,+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ None,+ ));+ for _ in 0..3 {+ let (_body, reply) = observed.recv().await.unwrap();+ let mut limited = response(429, "{}");+ limited.retry_after = Some("120".into());+ reply.send(limited).unwrap();+ }+ let (complete, not_before) = task.await.unwrap();+ assert!(!complete);+ let began = Instant::now();+ assert_eq!(not_before, Some(began + Duration::from_secs(120)));+ let state = Arc::new(std::sync::atomic::AtomicU8::new(1));+ let task_state = state.clone();+ let task = tokio::spawn(async move {+ settle_recovery_run(&task_state, complete, not_before, &TraceUploadCancel::new()).await;+ });+ tokio::task::yield_now().await;+ tokio::time::advance(MIN_INTERVAL).await;+ assert_eq!(+ state.load(Ordering::Acquire),+ 1,+ "must not rescan after only 60 seconds"+ );+ tokio::time::advance(MIN_INTERVAL).await;+ task.await.unwrap();+ assert_eq!(state.load(Ordering::Acquire), 0);+ assert_eq!(Instant::now(), began + Duration::from_secs(120));+}++#[test]+fn oversized_cursor_records_fail_closed_with_bounded_reads() {+ let fixture = Fixture::new();+ let path = fixture.write_session("large-cursor.jsonl", "large-cursor");+ record_agent_trace_outbox_upload(+ &fixture.agent_dir,+ &path,+ TraceUploadSignature::of(&path).unwrap(),+ )+ .unwrap();+ let entry = agent_trace_outbox_entry_path(&fixture.agent_dir, &path);+ let mut value: Value = serde_json::from_str(&std::fs::read_to_string(&entry).unwrap()).unwrap();+ value["padding"] = json!(" ".repeat(64 * 1024));+ std::fs::write(entry, value.to_string()).unwrap();+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ None+ );+}++#[tokio::test(start_paused = true)]+// Credential lookup must stay stable while synthetic background workers run.+#[allow(clippy::await_holding_lock)]+async fn replacement_at_capacity_reuses_a_slot_and_preserves_late_retirement() {+ let _env = env_lock();+ let _credentials = clear_trace_credentials();+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let original_path = fixture.write_session("original.jsonl", "original");+ let replacement_path = fixture.write_session("replacement.jsonl", "replacement");+ let source = controller(&fixture, &original_path, true);+ source.persisted(&original_path);+ let service = Service {+ controllers: Mutex::new(Vec::new()),+ wake: tokio::sync::Notify::new(),+ };+ assert!(service.register(&source));+ let mut hosts = vec![source.clone()];+ for n in 1..MAX_CONTROLLERS {+ let host = controller(+ &fixture,+ &fixture.session_dir.join(format!("host-{n}.jsonl")),+ false,+ );+ assert!(service.register(&host));+ hosts.push(host);+ }+ let old_sink = Arc::new(ScriptedTraceHttp::new(Vec::new()));+ let old_task = tokio::spawn(run_controller(+ Arc::downgrade(&source),+ Arc::new(tokio::sync::Semaphore::new(1)),+ old_sink.clone(),+ Some("http://synthetic.invalid".into()),+ ));+ let first = controller(&fixture, &replacement_path, true);+ assert!(service.replace(&source, &first));+ assert!(source.cancel.is_cancelled());+ old_task.await.unwrap();+ assert_eq!(old_sink.request_count(), 0);+ first.persisted(&replacement_path);+ let mut recovered = RecoveryRuns::new();+ let run = RecoveryRun::new(hosts.iter().map(Arc::downgrade).collect());+ run.state.store(2, Ordering::Release);+ recovered.insert(fixture.agent_dir.clone(), run);+ let snapshot = service.controllers.lock().unwrap().clone();+ retain_live_recoveries(&mut recovered, &snapshot);+ assert!(recovered[&fixture.agent_dir]+ .replay_needed+ .load(Ordering::Acquire));+ let latest = controller(&fixture, &replacement_path, true);+ assert!(service.replace(&first, &latest));+ assert!(first.cancel.is_cancelled());+ acknowledge_replacements(&mut service.controllers.lock().unwrap(), &snapshot);+ assert!(service+ .controllers+ .lock()+ .unwrap()+ .iter()+ .any(|registration| {+ registration.controller.ptr_eq(&Arc::downgrade(&latest))+ && registration.retired_predecessor+ }));+ let current = service.controllers.lock().unwrap().clone();+ retain_live_recoveries(&mut recovered, ¤t);+ acknowledge_replacements(&mut service.controllers.lock().unwrap(), ¤t);+ assert_eq!(service.controllers.lock().unwrap().len(), MAX_CONTROLLERS);+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));+ let (complete, _) = recover_with_backoff(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink.clone(),+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ Some(vec![Arc::downgrade(&latest)]),+ )+ .await;+ assert!(!complete); // Replacement owns its marker; the predecessor was recovered.+ assert_eq!(sink.request_count(), 1);+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &original_path),+ TraceUploadSignature::of(&original_path)+ );+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let task = tokio::spawn(run_controller(+ Arc::downgrade(&latest),+ Arc::new(tokio::sync::Semaphore::new(1)),+ Arc::new(ObservedSink {+ requests,+ cancelled: Arc::new(tokio::sync::Notify::new()),+ }),+ Some("http://synthetic.invalid".into()),+ ));+ let (_body, reply) = tokio::time::timeout(MIN_INTERVAL, observed.recv())+ .await+ .unwrap()+ .unwrap();+ reply.send(response(200, "{}")).unwrap();+ drop(latest);+ task.await.unwrap();+}++#[tokio::test(start_paused = true)]+async fn a_busy_delivery_lease_leaves_recovery_incomplete_until_release() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("leased-recovery.jsonl", "leased-recovery");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let lease = delivery_lease(&fixture.agent_dir, &path).unwrap();+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));+ assert!(+ !recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink.clone(),+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await,+ "a leased marker must keep the sweep incomplete so recovery re-arms"+ );+ drop(lease);+ assert!(+ recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await+ );+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ TraceUploadSignature::of(&path)+ );+}++#[tokio::test(start_paused = true)]+async fn exhausted_recovery_retries_leave_the_sweep_incomplete() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("exhausted-recovery.jsonl", "exhausted-recovery");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ // Three bounded sweep attempts each exhaust the transport retries.+ let sink = Arc::new(ScriptedTraceHttp::new(+ std::iter::repeat_n(Ok(response(503, "{}")), 12).collect(),+ ));+ assert!(+ !recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await,+ "work still due after the bounded retries must keep the sweep incomplete"+ );+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ None+ );+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());+}++#[tokio::test(start_paused = true)]+// The env lock is held across awaits by design (env must stay stable).+#[allow(clippy::await_holding_lock)]+async fn a_credentialless_controller_retries_after_credentials_arrive() {+ let _env = env_lock();+ let _credentials = clear_trace_credentials();+ let fixture = Fixture::new();+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);+ settings.set_agent_traces_enabled(true).unwrap();+ // Deliberately no auth.json yet: the delivery fails closed, and the live+ // controller must schedule its own retry rather than park forever.+ let path = fixture.write_session("late-login.jsonl", "late-login");+ let c = controller(&fixture, &path, true);+ c.persisted(&path);+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: Arc::new(tokio::sync::Notify::new()),+ });+ let task = tokio::spawn(run_controller(+ Arc::downgrade(&c),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ ));+ // Park so the controller really attempts at +1s against the missing+ // credentials before any write makes one deliverable.+ tokio::time::sleep(MIN_INTERVAL + DEBOUNCE * 2).await;+ std::fs::write(+ fixture.agent_dir.join("auth.json"),+ r#"{"prime-agent-traces":{"type":"api_key","key":"synthetic-only"}}"#,+ )+ .unwrap();+ let (_body, reply) = tokio::time::timeout(MIN_INTERVAL * 2, observed.recv())+ .await+ .expect("a credentialless delivery must retry after credentials arrive")+ .unwrap();+ reply.send(response(200, "{}")).unwrap();+ drop(c);+ task.await.unwrap();+}++#[tokio::test(start_paused = true)]+async fn a_settings_change_after_load_still_records_the_write_intent() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("settings-race.jsonl", "settings-race");+ let c = controller(&fixture, &path, true);+ // A concurrent project edit changes the settings metadata after the+ // host captured its consent snapshot: removal is unambiguous.+ std::fs::remove_file(fixture.agent_dir.join("settings.json")).unwrap();+ c.persisted(&path);+ assert!(+ agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists(),+ "the write intent must stay durable across a concurrent settings edit"+ );+ // The worker re-reads consent before delivery; restore the opted-in edit.+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);+ settings.set_agent_traces_enabled(true).unwrap();+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: Arc::new(tokio::sync::Notify::new()),+ });+ let task = tokio::spawn(run_controller(+ Arc::downgrade(&c),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ ));+ let (_body, reply) = tokio::time::timeout(MIN_INTERVAL, observed.recv())+ .await+ .expect("the worker revalidates consent and delivers the retained intent")+ .unwrap();+ reply.send(response(200, "{}")).unwrap();+ drop(c);+ task.await.unwrap();+}++#[tokio::test(start_paused = true)]+// The env lock is held across awaits by design (env must stay stable).+#[allow(clippy::await_holding_lock)]+async fn missing_credentials_leave_the_sweep_incomplete_until_provided() {+ let _env = env_lock();+ let _credentials = clear_trace_credentials();+ let fixture = Fixture::new();+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);+ settings.set_agent_traces_enabled(true).unwrap();+ // Deliberately no auth.json yet: the upload must fail closed.+ let path = fixture.write_session("no-credentials.jsonl", "no-credentials");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));+ assert!(+ !recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink.clone(),+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await,+ "missing credentials must keep the sweep incomplete so it re-arms"+ );+ std::fs::write(+ fixture.agent_dir.join("auth.json"),+ r#"{"prime-agent-traces":{"type":"api_key","key":"synthetic-only"}}"#,+ )+ .unwrap();+ assert!(+ recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await+ );+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ TraceUploadSignature::of(&path)+ );+}++#[tokio::test(start_paused = true)]+// Keep credential lookup stable across injected background I/O failures.+#[allow(clippy::await_holding_lock)]+async fn transient_recovery_reads_rearm_and_deliver_after_restoration() {+ let _env = env_lock();+ let _credentials = clear_trace_credentials();+ for stage in [+ RecoveryRead::Directory,+ RecoveryRead::Iteration,+ RecoveryRead::RecordMetadata,+ RecoveryRead::Record,+ RecoveryRead::SessionMetadata,+ RecoveryRead::Header,+ ] {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("transient.jsonl", "transient");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));+ let (complete, _) = recover_with_io(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink.clone(),+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ Some(Vec::new()),+ |read, _| {+ if read == stage {+ Err(std::io::Error::from(std::io::ErrorKind::Interrupted))+ } else {+ Ok(())+ }+ },+ )+ .await;+ assert!(!complete, "{stage:?} must keep recovery incomplete");+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());+ assert_eq!(sink.request_count(), 0);+ assert!(+ recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink.clone(),+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await+ );+ assert_eq!(sink.request_count(), 1);+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ TraceUploadSignature::of(&path)+ );+ }+}++#[tokio::test(start_paused = true)]+async fn terminal_invalid_records_and_headers_complete_without_network_or_rescan() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let valid = fixture.write_session("valid.jsonl", "valid");+ let header = std::fs::read_to_string(&valid).unwrap();+ for (name, bytes) in [+ ("malformed", b"not a header\n".to_vec()),+ ("utf8", vec![0xff, b'\n']),+ ("empty", Vec::new()),+ (+ "oversized",+ [+ header.lines().next().unwrap().as_bytes(),+ &vec![b' '; 256 * 1024],+ ]+ .concat(),+ ),+ ] {+ let path = fixture.session_dir.join(format!("{name}.jsonl"));+ std::fs::write(&path, bytes).unwrap();+ mark_pending(&fixture.agent_dir, &path).unwrap();+ }+ let outbox = agent_trace_outbox_dir(&fixture.agent_dir);+ std::fs::write(outbox.join("invalid-utf8.json"), [0xff]).unwrap();+ std::fs::write(outbox.join("oversized.json"), vec![b' '; 64 * 1024 + 1]).unwrap();+ std::fs::write(outbox.join("malformed.json"), "{").unwrap();+ let sink = Arc::new(ScriptedTraceHttp::new(Vec::new()));+ let (complete, not_before) = recover_with_backoff(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink.clone(),+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ Some(Vec::new()),+ )+ .await;+ assert_eq!((complete, not_before), (true, None));+ let state = std::sync::atomic::AtomicU8::new(1);+ settle_recovery_run(&state, complete, not_before, &TraceUploadCancel::new()).await;+ assert_eq!(state.load(Ordering::Acquire), 2);+ assert_eq!(sink.request_count(), 0);+}++#[tokio::test(start_paused = true)]+// The env lock is held across awaits to keep credential lookup deterministic.+#[allow(clippy::await_holding_lock)]+async fn bounded_admission_preserves_retirement_and_delivers_on_readmission() {+ let _env = env_lock();+ let _credentials = clear_trace_credentials();+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("admission.jsonl", "admission");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let service = Service {+ controllers: Mutex::new(Vec::new()),+ wake: tokio::sync::Notify::new(),+ };+ let mut hosts = Vec::new();+ for n in 0..MAX_CONTROLLERS {+ let host = controller(+ &fixture,+ &fixture.session_dir.join(format!("host-{n}.jsonl")),+ false,+ );+ assert!(service.register(&host));+ hosts.push(host);+ }+ let candidate = controller(&fixture, &path, true);+ assert!(!service.register(&candidate));+ assert_eq!(service.controllers.lock().unwrap().len(), MAX_CONTROLLERS);+ let mut recovered = RecoveryRuns::new();+ let run = RecoveryRun::new(hosts.iter().map(Arc::downgrade).collect());+ run.state.store(2, Ordering::Release);+ recovered.insert(fixture.agent_dir.clone(), run);+ drop(hosts.pop().unwrap());+ // Retirement cannot be discarded by foreground admission before replay sees it.+ assert!(!service.register(&candidate));+ let snapshot = service.controllers.lock().unwrap().clone();+ let retired = retain_live_recoveries(&mut recovered, &snapshot);+ assert!(recovered[&fixture.agent_dir]+ .replay_needed+ .load(Ordering::Acquire));+ assert_eq!(retired.len(), 1);+ acknowledge_retired_registrations(&mut service.controllers.lock().unwrap(), &retired);+ assert!(service.register(&candidate));+ assert_eq!(service.controllers.lock().unwrap().len(), MAX_CONTROLLERS);+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: Arc::new(tokio::sync::Notify::new()),+ });+ let task = tokio::spawn(run_controller(+ Arc::downgrade(&candidate),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ ));+ // Admission resumes the durable marker with no restart or fresh persist.+ let (_body, reply) = tokio::time::timeout(MIN_INTERVAL, observed.recv())+ .await+ .expect("readmitted controller must deliver existing intent")+ .unwrap();+ reply.send(response(200, "{}")).unwrap();+ drop(candidate);+ task.await.unwrap();+}++#[tokio::test(start_paused = true)]+async fn a_busy_delivery_lease_does_not_hold_upload_capacity() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let leased = fixture.write_session("lease-wait.jsonl", "lease-wait");+ let free = fixture.write_session("lease-free.jsonl", "lease-free");+ let busy_host = controller(&fixture, &leased, true);+ busy_host.persisted(&leased);+ let lease = delivery_lease(&fixture.agent_dir, &leased).unwrap();+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();+ let sink = Arc::new(ObservedSink {+ requests,+ cancelled: Arc::new(tokio::sync::Notify::new()),+ });+ let permits = Arc::new(tokio::sync::Semaphore::new(1));+ let busy_task = tokio::spawn(run_controller(+ Arc::downgrade(&busy_host),+ permits.clone(),+ sink.clone(),+ Some("http://synthetic.invalid".into()),+ ));+ // The busy host's due (+1s) must strictly precede the free host's (+1.5s):+ // startup re-persists each deadline from its own spawn instant, so the+ // ordering is built from distinct spawn instants, never wake order.+ tokio::time::sleep(Duration::from_millis(500)).await;+ let free_host = controller(&fixture, &free, true);+ free_host.persisted(&free);+ let free_task = tokio::spawn(run_controller(+ Arc::downgrade(&free_host),+ permits.clone(),+ sink,+ Some("http://synthetic.invalid".into()),+ ));+ // The busy host attempts the held lease at +1s and must release the shared+ // permit while it waits, or the free host only delivers at its +2s retry.+ let (body, reply) =+ tokio::time::timeout(DEBOUNCE + Duration::from_millis(250), observed.recv())+ .await+ .expect("a busy lease must not starve an unrelated delivery")+ .unwrap();+ assert!(+ body.contains("lease-free"),+ "the free session delivers first"+ );+ reply.send(response(200, "{}")).unwrap();+ drop(lease);+ let (body, reply) = tokio::time::timeout(MIN_INTERVAL, observed.recv())+ .await+ .expect("the leased session retries after the lease releases")+ .unwrap();+ assert!(body.contains("lease-wait"));+ reply.send(response(200, "{}")).unwrap();+ drop(busy_host);+ drop(free_host);+ busy_task.await.unwrap();+ free_task.await.unwrap();+}++#[tokio::test(start_paused = true)]+async fn pending_live_owner_keeps_recovery_incomplete_until_retirement() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("live-retirement.jsonl", "live-retirement");+ mark_pending(&fixture.agent_dir, &path).unwrap();+ let c = controller(&fixture, &path, true);+ let weak = Arc::downgrade(&c);+ let sink = Arc::new(ScriptedTraceHttp::new(vec![]));+ let (complete, _) = recover_with_backoff(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ Some(vec![weak]),+ )+ .await;+ assert!(!complete);+ drop(c);+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));+ assert!(+ recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await+ );+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ TraceUploadSignature::of(&path)+ );+}++#[tokio::test(start_paused = true)]+async fn host_retirement_rearms_an_already_complete_group_for_later_markers() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let path = fixture.write_session("late-marker.jsonl", "late-marker");+ let c = controller(&fixture, &path, true);+ let another = controller(&fixture, &fixture.session_dir.join("survivor.jsonl"), true);+ let weak = Arc::downgrade(&c);+ let another_weak = Arc::downgrade(&another);+ let mut runs = RecoveryRuns::new();+ let run = RecoveryRun::new(vec![weak.clone(), another_weak.clone()]);+ run.state.store(2, Ordering::Release);+ runs.insert(fixture.agent_dir.clone(), run);+ let registration = Registration::of(&c);+ let another_registration = Registration::of(&another);+ c.persisted(&path);+ drop(c);+ let _ = retain_live_recoveries(&mut runs, &[registration, another_registration]);+ let run = runs.get(&fixture.agent_dir).unwrap();+ assert!(run.replay_needed.load(Ordering::Acquire));+ assert!(!run.cancel.is_cancelled());+ rearm_retired_recovery(run);+ assert_eq!(run.state.load(Ordering::Acquire), 1);+ tokio::task::yield_now().await;+ tokio::time::advance(MIN_INTERVAL).await;+ tokio::task::yield_now().await;+ assert_eq!(run.state.load(Ordering::Acquire), 0);+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));+ assert!(+ recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await+ );+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),+ TraceUploadSignature::of(&path)+ );+}++#[tokio::test(start_paused = true)]+async fn retirement_racing_completion_survives_and_preserves_the_retry_deadline() {+ let run = RecoveryRun::new(vec![]);+ run.state.store(1, Ordering::Release);+ let complete = !run.replay_needed.load(Ordering::Acquire);+ run.replay_needed.store(true, Ordering::Release);+ let deadline = Instant::now() + Duration::from_secs(120);+ *run.not_before.lock().unwrap() = Some(deadline);+ settle_recovery_run(&run.state, complete, Some(deadline), &run.cancel).await;+ assert_eq!(run.state.load(Ordering::Acquire), 2);+ rearm_retired_recovery(&run);+ rearm_retired_recovery(&run);+ assert_eq!(run.state.load(Ordering::Acquire), 1);+ tokio::task::yield_now().await;+ tokio::time::advance(MIN_INTERVAL).await;+ assert_eq!(run.state.load(Ordering::Acquire), 1);+ tokio::time::advance(MIN_INTERVAL).await;+ tokio::task::yield_now().await;+ assert_eq!(run.state.load(Ordering::Acquire), 0);+ assert!(+ run.replay_needed.load(Ordering::Acquire),+ "only a new sweep may clear retirement intent"+ );+}++#[tokio::test(start_paused = true)]+async fn between_tick_retirement_and_late_acknowledgement_preserve_pending_markers() {+ let fixture = Fixture::new();+ enable_synthetic_fixture(&fixture);+ let survivor = controller(&fixture, &fixture.session_dir.join("survivor.jsonl"), true);+ let run = RecoveryRun::new(vec![Arc::downgrade(&survivor)]);+ run.state.store(2, Ordering::Release);+ let mut runs = RecoveryRuns::new();+ runs.insert(fixture.agent_dir.clone(), run);+ let mut registrations = vec![Registration::of(&survivor)];+ let first_path = fixture.write_session("between-ticks.jsonl", "between-ticks");+ let first = controller(&fixture, &first_path, true);+ registrations.push(Registration::of(&first));+ first.persisted(&first_path);+ drop(first);+ let retired = retain_live_recoveries(&mut runs, ®istrations);+ assert_eq!(retired.len(), 1);+ let second_path = fixture.write_session("after-snapshot.jsonl", "after-snapshot");+ let second = controller(&fixture, &second_path, true);+ registrations.push(Registration::of(&second));+ second.persisted(&second_path);+ drop(second);+ acknowledge_retired_registrations(&mut registrations, &retired);+ assert_eq!(+ registrations.len(),+ 2,+ "a retirement after the snapshot must remain until the next tick"+ );+ let retired = retain_live_recoveries(&mut runs, ®istrations);+ assert_eq!(retired.len(), 1);+ acknowledge_retired_registrations(&mut registrations, &retired);+ assert_eq!(registrations.len(), 1);+ let run = runs.get(&fixture.agent_dir).unwrap();+ assert!(run.replay_needed.load(Ordering::Acquire));+ rearm_retired_recovery(run);+ tokio::task::yield_now().await;+ tokio::time::advance(MIN_INTERVAL).await;+ tokio::task::yield_now().await;+ assert_eq!(run.state.load(Ordering::Acquire), 0);+ let sink = Arc::new(ScriptedTraceHttp::new(vec![+ Ok(response(200, "{}")),+ Ok(response(200, "{}")),+ ]));+ assert!(+ recover(+ fixture.cwd.clone(),+ fixture.agent_dir.clone(),+ Arc::new(tokio::sync::Semaphore::new(1)),+ sink,+ Some("http://synthetic.invalid".into()),+ TraceUploadCancel::new(),+ )+ .await+ );+ for path in [&first_path, &second_path] {+ assert_eq!(+ read_agent_trace_outbox_entry(&fixture.agent_dir, path),+ TraceUploadSignature::of(path)+ );+ }+}diff --git a/crates/pa-core/src/agent_traces/tests.rs b/crates/pa-core/src/agent_traces/tests.rsindex 9504fce826..988fc34615 100644--- a/crates/pa-core/src/agent_traces/tests.rs+++ b/crates/pa-core/src/agent_traces/tests.rs@@ -14,13 +14,13 @@ type CapturedRequest = (String, Vec<(String, String)>, String);/// A scripted transport: one PUT slot at a time with its captured/// request and scripted answer.-struct ScriptedTraceHttp {+pub(super) struct ScriptedTraceHttp {requests: Mutex<Vec<CapturedRequest>>,answers: Mutex<VecDeque<Result<TraceHttpResponse, TraceHttpError>>>,}impl ScriptedTraceHttp {- fn new(answers: Vec<Result<TraceHttpResponse, TraceHttpError>>) -> Self {+ pub(super) fn new(answers: Vec<Result<TraceHttpResponse, TraceHttpError>>) -> Self {ScriptedTraceHttp {requests: Mutex::new(Vec::new()),answers: Mutex::new(answers.into_iter().collect()),@@ -30,6 +30,10 @@ impl ScriptedTraceHttp {fn last_request(&self) -> CapturedRequest {self.requests.lock().unwrap().last().cloned().unwrap()}++ pub(super) fn request_count(&self) -> usize {+ self.requests.lock().unwrap().len()+ }}impl TraceHttp for ScriptedTraceHttp {@@ -48,7 +52,7 @@ impl TraceHttp for ScriptedTraceHttp {}}-fn response(status: u16, body: &str) -> TraceHttpResponse {+pub(super) fn response(status: u16, body: &str) -> TraceHttpResponse {TraceHttpResponse {status,body: body.to_string(),@@ -56,17 +60,17 @@ fn response(status: u16, body: &str) -> TraceHttpResponse {}}-struct Fixture {+pub(super) struct Fixture {/// The temp dir stays alive for the fixture's life (the paths/// point into it); it is never read._dir: tempfile::TempDir,- cwd: PathBuf,- agent_dir: PathBuf,- session_dir: PathBuf,+ pub(super) cwd: PathBuf,+ pub(super) agent_dir: PathBuf,+ pub(super) session_dir: PathBuf,}impl Fixture {- fn new() -> Self {+ pub(super) fn new() -> Self {let dir = tempfile::tempdir().expect("temp dir");let agent_dir = dir.path().join("agent");let session_dir = agent_dir.join("sessions");@@ -79,7 +83,7 @@ impl Fixture {}}- fn write_session(&self, name: &str, id: &str) -> PathBuf {+ pub(super) fn write_session(&self, name: &str, id: &str) -> PathBuf {let path = self.session_dir.join(name);std::fs::write(&path,@@ -113,12 +117,42 @@ impl Fixture {/// The engine reads process env (the credential keys); the tests that/// touch it serialize on one lock.-fn env_lock() -> std::sync::MutexGuard<'static, ()> {+pub(super) fn env_lock() -> std::sync::MutexGuard<'static, ()> {static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner)}+/// Clears the credential env vars under the lock and restores the prior+/// values (or absence) on drop — including on assertion panics.+pub(super) struct TraceEnvCredentials {+ saved: [(&'static str, Option<std::ffi::OsString>); 2],+}++pub(super) fn clear_trace_credentials() -> TraceEnvCredentials {+ let saved = [+ (+ "PRIME_AGENT_TRACES_API_KEY",+ std::env::var_os("PRIME_AGENT_TRACES_API_KEY"),+ ),+ ("PRIME_API_KEY", std::env::var_os("PRIME_API_KEY")),+ ];+ std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");+ std::env::remove_var("PRIME_API_KEY");+ TraceEnvCredentials { saved }+}++impl Drop for TraceEnvCredentials {+ fn drop(&mut self) {+ for (name, value) in &self.saved {+ match value {+ Some(value) => std::env::set_var(name, value),+ None => std::env::remove_var(name),+ }+ }+ }+}+#[test]fn the_header_validation_matches_ts() {let fixture = Fixture::new();@@ -143,8 +177,7 @@ fn the_header_validation_matches_ts() {#[allow(clippy::await_holding_lock)]async fn the_upload_sends_the_ts_request_and_records_the_cursor() {let _env = env_lock();- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");- std::env::remove_var("PRIME_API_KEY");+ let _credentials = clear_trace_credentials();let fixture = Fixture::new();let session = fixture.write_session("s.jsonl", "sid-1");std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");@@ -186,7 +219,6 @@ async fn the_upload_sends_the_ts_request_and_records_the_cursor() {assert!(signature_equals(recorded, signature));let log = std::fs::read_to_string(agent_traces_log_path(&fixture.agent_dir)).expect("log");assert!(log.contains("uploaded session sid-1 (42 bytes)"), "{log}");- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");}#[tokio::test]@@ -212,8 +244,7 @@ async fn the_disabled_requirement_gate_matches_ts() {#[allow(clippy::await_holding_lock)]async fn a_missing_credential_short_circuits_the_request() {let _env = env_lock();- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");- std::env::remove_var("PRIME_API_KEY");+ let _credentials = clear_trace_credentials();let fixture = Fixture::new();let session = fixture.write_session("s.jsonl", "sid");let http = ScriptedTraceHttp::new(vec![]);@@ -226,6 +257,7 @@ async fn a_missing_credential_short_circuits_the_request() {#[allow(clippy::await_holding_lock)]async fn an_oversize_session_reports_the_limit() {let _env = env_lock();+ let _credentials = clear_trace_credentials();std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");let fixture = Fixture::new();let session = fixture.session_dir.join("big.jsonl");@@ -251,7 +283,6 @@ async fn an_oversize_session_reports_the_limit() {max_bytes: MAX_TRACE_BYTES,});- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");}#[tokio::test]@@ -259,6 +290,7 @@ async fn an_oversize_session_reports_the_limit() {#[allow(clippy::await_holding_lock)]async fn an_error_response_carries_the_status_and_message() {let _env = env_lock();+ let _credentials = clear_trace_credentials();std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");let fixture = Fixture::new();let session = fixture.write_session("s.jsonl", "sid");@@ -272,7 +304,6 @@ async fn an_error_response_carries_the_status_and_message() {retry_after_ms: None,});- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");}#[tokio::test]@@ -280,6 +311,7 @@ async fn an_error_response_carries_the_status_and_message() {#[allow(clippy::await_holding_lock)]async fn the_retriable_statuses_back_off_and_503_honors_retry_after() {let _env = env_lock();+ let _credentials = clear_trace_credentials();std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");let fixture = Fixture::new();let session = fixture.write_session("s.jsonl", "sid");@@ -306,7 +338,6 @@ async fn the_retriable_statuses_back_off_and_503_honors_retry_after() {assert!(matches!(result, TraceUploadResult::Uploaded { .. }));// The Retry-After second won over the exponential backoff.assert_eq!(delays.load(Ordering::SeqCst), 1000);- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");}#[tokio::test]@@ -314,6 +345,7 @@ async fn the_retriable_statuses_back_off_and_503_honors_retry_after() {#[allow(clippy::await_holding_lock)]async fn the_outbox_cursor_makes_an_enabled_upload_unchanged() {let _env = env_lock();+ let _credentials = clear_trace_credentials();std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");let fixture = Fixture::new();let session = fixture.write_session("s.jsonl", "sid");@@ -331,7 +363,6 @@ async fn the_outbox_cursor_makes_an_enabled_upload_unchanged() {options.require_enabled = true;let result = upload_trace_file(&options).await;assert_eq!(result, TraceUploadResult::Unchanged);- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");}#[tokio::test]@@ -485,6 +516,7 @@ fn format_http_date(ms: u64) -> String {#[allow(clippy::await_holding_lock)]async fn the_upload_all_sweeps_with_the_gate_and_tallies() {let _env = env_lock();+ let _credentials = clear_trace_credentials();std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");let fixture = Fixture::new();fixture.write_session("a.jsonl", "sid-a");@@ -525,7 +557,6 @@ async fn the_upload_all_sweeps_with_the_gate_and_tallies() {vec![(0, 1), (1, 1)],"the progress reports completion in file order");- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");}#[tokio::test(start_paused = true)]@@ -533,6 +564,7 @@ async fn the_upload_all_sweeps_with_the_gate_and_tallies() {#[allow(clippy::await_holding_lock)]async fn the_rate_gate_reports_the_wait_and_serializes() {let _env = env_lock();+ let _credentials = clear_trace_credentials();std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");let fixture = Fixture::new();let http = ScriptedTraceHttp::new(vec![]);@@ -563,7 +595,6 @@ async fn the_rate_gate_reports_the_wait_and_serializes() {.await.expect("the second slot");assert!(delays.load(Ordering::SeqCst) >= TRACE_UPLOAD_ALL_MIN_REQUEST_INTERVAL_MS - 1000);- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");}#[tokio::test]@@ -571,6 +602,7 @@ async fn the_rate_gate_reports_the_wait_and_serializes() {#[allow(clippy::await_holding_lock)]async fn the_cancel_stops_the_sweep_between_files() {let _env = env_lock();+ let _credentials = clear_trace_credentials();std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");let fixture = Fixture::new();fixture.write_session("a.jsonl", "sid-a");@@ -596,7 +628,6 @@ async fn the_cancel_stops_the_sweep_between_files() {assert_eq!(result.total, 2);assert_eq!(result.uploaded, 0);assert_eq!(result.skipped, 2);- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");}#[test]@@ -623,3 +654,29 @@ fn the_outbox_entry_path_is_the_path_hash() {path);}++#[tokio::test(start_paused = true)]+async fn consent_revoked_during_backoff_prevents_the_next_request() {+ let fixture = Fixture::new();+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);+ settings.set_agent_traces_enabled(true).unwrap();+ let http = ScriptedTraceHttp::new(vec![Ok(response(500, "synthetic failure"))]);+ let cwd = fixture.cwd.clone();+ let agent_dir = fixture.agent_dir.clone();+ let mut options = fixture.options(&http, None);+ options.require_enabled = true;+ options.on_upload_delay = Some(Arc::new(move |_| {+ let mut settings = crate::settings::SettingsManager::create(&cwd, &agent_dir);+ settings.set_agent_traces_enabled(false).unwrap();+ }));+ let result = super::upload::fetch_with_retry(+ &options,+ "http://synthetic.invalid",+ vec![],+ String::new(),+ None,+ )+ .await;+ assert_eq!(result, Err(TraceHttpError::Cancelled));+ assert_eq!(http.requests.lock().unwrap().len(), 1);+}diff --git a/crates/pa-core/src/agent_traces/upload.rs b/crates/pa-core/src/agent_traces/upload.rsindex fd27a8cba4..ca657f1550 100644--- a/crates/pa-core/src/agent_traces/upload.rs+++ b/crates/pa-core/src/agent_traces/upload.rs@@ -37,7 +37,7 @@ impl TraceUploadOptions<'_> {if self.reload_config {let _ = settings.reload();}- settings.get_agent_traces_enabled()+ settings.errors().is_empty() && settings.get_agent_traces_enabled()}}@@ -93,16 +93,33 @@ pub(super) async fn perform_agent_trace_upload(if options.require_enabled && !options.enabled() {return TraceUploadResult::Disabled;}- let body = match tokio::fs::read_to_string(session_file).await {+ // Bound the read itself: an append racing the initial stat must not allocate+ // an arbitrarily large body or bypass the server's 20 MiB limit.+ let read_body = async {+ use tokio::io::AsyncReadExt;+ let file = tokio::fs::File::open(session_file).await?;+ let mut body = String::new();+ file.take(MAX_TRACE_BYTES + 1)+ .read_to_string(&mut body)+ .await?;+ Ok::<_, std::io::Error>(body)+ };+ let body = match read_body.await {Ok(body) => body,Err(error) => {return TraceUploadResult::Failed {status_code: None,message: error.to_string(),retry_after_ms: None,- }+ };}};+ if body.len() as u64 > MAX_TRACE_BYTES {+ return TraceUploadResult::TooLarge {+ size: body.len() as u64,+ max_bytes: MAX_TRACE_BYTES,+ };+ }if body.trim().is_empty() {return TraceUploadResult::EmptySession;}@@ -151,7 +168,7 @@ pub(super) async fn perform_agent_trace_upload(status_code: None,message: error.message(),retry_after_ms: None,- }+ };}};if !(200..300).contains(&response.status) {@@ -193,7 +210,7 @@ pub(super) async fn perform_agent_trace_upload(/// The gate runs before every attempt, the retriable statuses/network/// errors back off with jitter, and 503 honors `Retry-After`.-async fn fetch_with_retry(+pub(super) async fn fetch_with_retry(options: &TraceUploadOptions<'_>,url: &str,headers: Vec<(String, String)>,@@ -210,6 +227,9 @@ async fn fetch_with_retry(if options.cancel.is_some_and(TraceUploadCancel::is_cancelled) {return Err(TraceHttpError::Cancelled);}+ if options.require_enabled && !options.enabled() {+ return Err(TraceHttpError::Cancelled);+ }let mut retry_delay_ms: Option<u64> = None;let result = options.httpdiff --git a/crates/pa-core/src/platform/lock_dir.rs b/crates/pa-core/src/platform/lock_dir.rsindex 15c0977a83..c37ad39700 100644--- a/crates/pa-core/src/platform/lock_dir.rs+++ b/crates/pa-core/src/platform/lock_dir.rs@@ -486,6 +486,30 @@ impl Drop for LockDir {}}+/// Advisory exclusive lock on an open file, released on drop or process+/// death — the platform-walled form of `std::fs::File::lock`. OS file locks+/// (crash-released, no stale-lock reclaim) live here so platform-specific+/// behavior stays auditable in one place, like the directory locks above.+///+/// # Errors+///+/// `WouldBlock` (blocking form: after interruption) never applies; other I/O+/// errors as the platform reports them.+pub fn lock_exclusive(file: &std::fs::File) -> io::Result<()> {+ file.lock()+}++/// Nonblocking [`lock_exclusive`]: any held lock (fresh contention or I/O+/// failure) surfaces as an error carrying the platform's try-lock error.+///+/// # Errors+///+/// The platform's [`std::fs::TryLockError`] (contention included) wrapped in+/// an I/O error; nothing is returned on success.+pub fn try_lock_exclusive(file: &std::fs::File) -> io::Result<()> {+ file.try_lock().map_err(io::Error::other)+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/pa-core/src/platform/mod.rs b/crates/pa-core/src/platform/mod.rsindex 4cecd29760..1b0f6bda9c 100644--- a/crates/pa-core/src/platform/mod.rs+++ b/crates/pa-core/src/platform/mod.rs@@ -10,7 +10,7 @@ pub mod process;pub mod shell;pub use fs::fsync;-pub use lock_dir::LockDir;+pub use lock_dir::{lock_exclusive, try_lock_exclusive, LockDir};pub use perms::{file_mode, is_executable, is_readable_writable, restrict_dir, restrict_file, set_private_mode,};diff --git a/crates/pa-core/src/session_engine/engine.rs b/crates/pa-core/src/session_engine/engine.rsindex b1123bfcb8..4d0fa4a45c 100644--- a/crates/pa-core/src/session_engine/engine.rs+++ b/crates/pa-core/src/session_engine/engine.rs@@ -174,7 +174,7 @@ pub async fn create_session(mut config: SessionEngineConfig) -> anyhow::Result<Slet cwd = config.cwd.clone();// Session persistence first: the conversation-log path and the resume// context both come from the session manager.- let session_manager = config+ let mut session_manager = config.session_manager.unwrap_or_else(|| SessionManager::in_memory(&cwd));let conversation_log = {@@ -189,6 +189,22 @@ pub async fn create_session(mut config: SessionEngineConfig) -> anyhow::Result<S.map(|path| path.display().to_string())})};+ let (settings, trace_consent) =+ crate::agent_traces::ContinuousTraceUpload::load_settings(&cwd, &config.agent_dir);+ let traces = session_manager+ .is_persisted()+ .then(|| {+ crate::agent_traces::ContinuousTraceUpload::install(+ &cwd,+ &config.agent_dir,+ session_manager.get_session_file(),+ trace_consent,+ )+ })+ .flatten();+ if let Some(traces) = traces {+ session_manager.on_persist(Box::new(move |path| traces.persisted(path)));+ }let wiring = super::runtime_wiring::wire_session_runtime(session_manager,&config.agent_dir,@@ -200,7 +216,6 @@ pub async fn create_session(mut config: SessionEngineConfig) -> anyhow::Result<Sconfig.cron_store.clone(),);- let settings = crate::settings::SettingsManager::create(&cwd, &config.agent_dir);let service_tier_preference = settings.get_default_service_tier();// Captured before `settings` moves into the resource loader: the// compaction budget and the auto-refine gates.diff --git a/crates/pa-daemon/src/branch_navigation.rs b/crates/pa-daemon/src/branch_navigation.rsindex a873a5c8a3..b8761e2197 100644--- a/crates/pa-daemon/src/branch_navigation.rs+++ b/crates/pa-daemon/src/branch_navigation.rs@@ -390,6 +390,10 @@ impl TreeNavigation {let file = session_dir.join(crate::session_store::session_file_name(forked.session_id()));forked.set_path(file);+ forked.trace_upload = store+ .trace_upload+ .as_ref()+ .and_then(|traces| traces.forked(&forked.path));if let Some(lease) = &store.lease {forked.lease =Some(lease.acquire_target(&forked.path).map_err(|error| {@@ -429,7 +433,7 @@ impl TreeNavigation {/// `fork`'s swap phase (TS `buildAndApplyReplacement`): the store, the engine's session/// file, and the rebuilt context move onto the prepared fork file; the teardown runs/// between the prepare and this swap.- pub(crate) async fn replace_with_fork(&self, forked: SessionFile) -> Result<(), String> {+ pub(crate) async fn replace_with_fork(&self, mut forked: SessionFile) -> Result<(), String> {let branch_entries = forked.branch_file_entries();let new_path = forked.path.clone();// Prime the fork store's usage fold before it enters the core: the@@ -439,10 +443,30 @@ impl TreeNavigation {let _ =tokio::task::spawn_blocking(move || crate::session_store::read_session_info(&primed)).await;- let previous = {+ let (previous, transferred) = {let mut core = self.core.lock().unwrap();- core.store.replace(forked)+ // A navigating fork replaces its source: when the prepare could not+ // admit a distinct installation (registry at capacity), the+ // publication takes the predecessor's slot. No fallible work sits+ // between the transfer and the store swap.+ let mut transferred = None;+ if forked.trace_upload.is_none() {+ transferred = core+ .store+ .as_ref()+ .and_then(|old| old.trace_upload.as_ref())+ .and_then(|traces| {+ traces.rebind(std::path::Path::new(&core.cwd), &forked.path)+ });+ forked.trace_upload.clone_from(&transferred);+ }+ (core.store.replace(forked), transferred)};+ if let Some(traces) = &transferred {+ // The fork was rewritten before its controller existed; the+ // pre-written file owes a delivery only after publication.+ traces.persisted(&new_path);+ }// The old store's lease release flushes the window and info// sidecars (megabytes for a large session): off the core lock// and the runtime.@@ -603,3 +627,133 @@ impl Worker {}}}++#[cfg(test)]+mod trace_fork_tests {+ use super::*;++ #[tokio::test]+ async fn below_capacity_fork_keeps_its_prepared_controller() {+ let _env = crate::trace_test_env::lock_env();+ let dir = tempfile::tempdir().unwrap();+ let agent_dir = dir.path().join("agent");+ let session_dir = agent_dir.join("sessions");+ std::fs::create_dir_all(&session_dir).unwrap();+ let mut settings = pa_core::settings::SettingsManager::create(dir.path(), &agent_dir);+ settings.set_agent_traces_enabled(true).unwrap();+ let old_path = session_dir.join("old.jsonl");+ let (_, consent) =+ pa_core::agent_traces::ContinuousTraceUpload::load_settings(dir.path(), &agent_dir);+ let controller = pa_core::agent_traces::ContinuousTraceUpload::install(+ dir.path(),+ &agent_dir,+ Some(&old_path),+ consent,+ )+ .unwrap();+ let mut store = SessionFile::create(dir.path().to_string_lossy().as_ref(), None, 0);+ store.set_path(old_path.clone());+ store.trace_upload = Some(controller.clone());+ let core = Arc::new(Mutex::new(SessionCore::test_core(+ Some(store),+ dir.path().to_string_lossy().into_owned(),+ )));+ let navigation = TreeNavigation::new(+ Arc::new(crate::engine::ScriptedEngine::default()),+ core.clone(),+ Arc::new(tokio::sync::Notify::new()),+ );+ // A spare registry slot admits the fork's distinct controller at prepare+ // time; publication must keep it attached for later writes.+ let (_, fork_consent) =+ pa_core::agent_traces::ContinuousTraceUpload::load_settings(dir.path(), &agent_dir);+ let prepared = pa_core::agent_traces::ContinuousTraceUpload::install(+ dir.path(),+ &agent_dir,+ Some(&session_dir.join("fork.jsonl")),+ fork_consent,+ )+ .unwrap();+ let mut forked = SessionFile::create(dir.path().to_string_lossy().as_ref(), None, 0);+ forked.set_path(session_dir.join("fork.jsonl"));+ forked.trace_upload = Some(prepared.clone());+ forked.rewrite().unwrap();+ navigation.replace_with_fork(forked).await.unwrap();+ let live = core.lock().unwrap();+ let published = live.store.as_ref().unwrap();+ let hook = published+ .trace_upload+ .as_ref()+ .expect("the prepared hook must survive");+ assert!(+ Arc::ptr_eq(hook, &prepared),+ "publication must keep the prepared controller attached"+ );+ drop(live);+ let outbox = agent_dir.join("agent-traces-outbox");+ let markers = std::fs::read_dir(&outbox)+ .expect("the rewritten fork file owes a delivery")+ .filter_map(std::result::Result::ok)+ .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json"))+ .count();+ assert_eq!(markers, 1);+ }++ #[tokio::test]+ async fn fork_at_capacity_publishes_with_the_predecessors_slot() {+ let _env = crate::trace_test_env::lock_env();+ let dir = tempfile::tempdir().unwrap();+ let agent_dir = dir.path().join("agent");+ let session_dir = agent_dir.join("sessions");+ std::fs::create_dir_all(&session_dir).unwrap();+ let mut settings = pa_core::settings::SettingsManager::create(dir.path(), &agent_dir);+ settings.set_agent_traces_enabled(true).unwrap();+ let old_path = session_dir.join("old.jsonl");+ let (_, consent) =+ pa_core::agent_traces::ContinuousTraceUpload::load_settings(dir.path(), &agent_dir);+ let controller = pa_core::agent_traces::ContinuousTraceUpload::install(+ dir.path(),+ &agent_dir,+ Some(&old_path),+ consent,+ )+ .unwrap();+ let mut store = SessionFile::create(dir.path().to_string_lossy().as_ref(), None, 0);+ store.set_path(old_path.clone());+ store.trace_upload = Some(controller.clone());+ let core = Arc::new(Mutex::new(SessionCore::test_core(+ Some(store),+ dir.path().to_string_lossy().into_owned(),+ )));+ let navigation = TreeNavigation::new(+ Arc::new(crate::engine::ScriptedEngine::default()),+ core.clone(),+ Arc::new(tokio::sync::Notify::new()),+ );+ // At capacity the prepare-time admission is rejected (the registry+ // bound is pa-core's, covered by its bounded-admission tests); the+ // unhooked prepared fork is that rejection's daemon-side shape.+ let mut forked = SessionFile::create(dir.path().to_string_lossy().as_ref(), None, 0);+ forked.set_path(session_dir.join("fork.jsonl"));+ forked.rewrite().unwrap();+ navigation.replace_with_fork(forked).await.unwrap();+ let live = core.lock().unwrap();+ let published = live.store.as_ref().unwrap();+ assert!(+ published.trace_upload.is_some(),+ "a replacing fork must publish with its predecessor's slot"+ );+ drop(live);+ let outbox = agent_dir.join("agent-traces-outbox");+ let markers: Vec<_> = std::fs::read_dir(&outbox)+ .expect("the pre-written fork file owes a delivery")+ .map(|entry| entry.expect("outbox entry").path())+ .filter(|path| path.extension().is_some_and(|ext| ext == "json"))+ .collect();+ assert_eq!(+ markers.len(),+ 1,+ "exactly the fork's pending marker must be recorded"+ );+ }+}diff --git a/crates/pa-daemon/src/lib.rs b/crates/pa-daemon/src/lib.rsindex f45c48d13c..05686a3c22 100644--- a/crates/pa-daemon/src/lib.rs+++ b/crates/pa-daemon/src/lib.rs@@ -126,3 +126,5 @@ pub(crate) mod worker_stderr;#[cfg(test)]pub(crate) mod test_support;+#[cfg(test)]+pub(crate) mod trace_test_env;diff --git a/crates/pa-daemon/src/session_navigation.rs b/crates/pa-daemon/src/session_navigation.rsindex 59c2bd217e..965c071c3a 100644--- a/crates/pa-daemon/src/session_navigation.rs+++ b/crates/pa-daemon/src/session_navigation.rs@@ -17,6 +17,8 @@ use crate::worker::{SessionCore, Worker};pub(crate) struct PreparedReplacement {pub(crate) file: SessionFile,pub(crate) cwd: Option<String>,+ // A successful fresh write owes scheduling only after publication.+ trace_persisted: bool,}/// The navigation surface: the prepare and swap phases of the replacement@@ -35,7 +37,12 @@ impl SessionNavigation {/// Swap the live session onto `file` (store, engine session file, rebuilt/// context). The caller retires the previous runtime and rebinds the cwd/// first, so the context park lands on the fresh session.- async fn replace_session(&self, file: SessionFile) -> Result<(), String> {+ async fn replace_session(&self, target: PreparedReplacement) -> Result<(), String> {+ let PreparedReplacement {+ mut file,+ trace_persisted,+ ..+ } = target;let branch_entries = file.branch_file_entries();let new_path = file.path.clone();// Prime the new store's usage fold before it enters the core: the@@ -44,10 +51,24 @@ impl SessionNavigation {let _ =tokio::task::spawn_blocking(move || crate::session_store::read_session_info(&primed)).await;- let previous = {+ let (previous, traces) = {let mut core = self.core.lock().unwrap();- core.store.replace(file)+ if file.trace_upload.is_none() {+ file.trace_upload = core+ .store+ .as_ref()+ .and_then(|old| old.trace_upload.as_ref())+ .and_then(|traces| traces.rebind(std::path::Path::new(&core.cwd), &file.path));+ }+ let traces = file.trace_upload.clone();+ // No await or fallible preparation between slot transfer and publication.+ (core.store.replace(file), traces)};+ if let Some(traces) = traces.filter(|_| trace_persisted) {+ // The successful prepare wrote this file before its controller existed.+ // Record bounded intent outside the core lock, after publication.+ traces.persisted(&new_path);+ }// The old store's lease release flushes the window and info// sidecars (megabytes for a large session): off the core lock// and the runtime.@@ -131,9 +152,11 @@ impl SessionNavigation {}}// TS `newSession` keeps the runtime's cwd: the fresh session runs where the live one did.+ let trace_persisted = !fresh.path.as_os_str().is_empty();Ok(PreparedReplacement {file: fresh,cwd: None,+ trace_persisted,})}@@ -274,7 +297,11 @@ impl SessionNavigation {));}}- Ok(PreparedReplacement { file, cwd })+ Ok(PreparedReplacement {+ file,+ cwd,+ trace_persisted: false,+ })}}@@ -316,7 +343,7 @@ impl Worker {if let Some(cwd) = target.cwd.as_deref() {self.rebind_worker_cwd(cwd);}- match self.navigation.replace_session(target.file).await {+ match self.navigation.replace_session(target).await {Ok(()) => {self.refresh_replaced_session_state().await;self.reseed_service_tier_for_replacement();@@ -870,3 +897,51 @@ mod tests {}}}++#[cfg(test)]+mod trace_replacement_tests {+ use super::*;++ #[test]+ fn failed_new_session_prepare_preserves_the_live_trace_registration() {+ let _env = crate::trace_test_env::lock_env();+ let dir = tempfile::tempdir().unwrap();+ let blocked = dir.path().join("not-a-directory");+ std::fs::write(&blocked, "synthetic blocker").unwrap();+ let old_path = blocked.join("old.jsonl");+ let agent_dir = dir.path().join("agent");+ let (_, consent) =+ pa_core::agent_traces::ContinuousTraceUpload::load_settings(dir.path(), &agent_dir);+ let controller = pa_core::agent_traces::ContinuousTraceUpload::install(+ dir.path(),+ &agent_dir,+ Some(&old_path),+ consent,+ )+ .unwrap();+ let mut store = SessionFile::create(dir.path().to_str().unwrap(), None, 0);+ store.set_path(old_path.clone());+ store.trace_upload = Some(controller.clone());+ let core = Arc::new(Mutex::new(SessionCore::test_core(+ Some(store),+ dir.path().to_string_lossy().into_owned(),+ )));+ let navigation = SessionNavigation::new(+ Arc::new(crate::engine::ScriptedEngine::default()),+ core.clone(),+ );+ assert!(navigation.prepare_new_session(&json!({})).is_err());+ let live = core.lock().unwrap();+ assert_eq!(live.store.as_ref().unwrap().path, old_path);+ assert!(Arc::ptr_eq(+ live.store.as_ref().unwrap().trace_upload.as_ref().unwrap(),+ &controller+ ));+ drop(live);+ // A cancelled predecessor cannot transfer its registry slot again.+ assert!(controller+ .rebind(dir.path(), &dir.path().join("probe.jsonl"))+ .is_some());+ assert!(!agent_dir.join("agent-traces-outbox").exists());+ }+}diff --git a/crates/pa-daemon/src/session_store.rs b/crates/pa-daemon/src/session_store.rsindex 6c3e0e7ee1..d0197d818a 100644--- a/crates/pa-daemon/src/session_store.rs+++ b/crates/pa-daemon/src/session_store.rs@@ -122,6 +122,7 @@ pub struct SessionFile {pub(crate) by_id: HashMap<String, usize>,pub(crate) leaf_id: Option<String>,pub(crate) window: Option<SessionWindow>,+ pub(crate) trace_upload: Option<std::sync::Arc<pa_core::agent_traces::ContinuousTraceUpload>>,pub(crate) lease: Option<std::sync::Arc<crate::lease::SessionLease>>,/// Rows the full reader skipped as unparsable (the TS loader's/// lenient skip): zero on a healthy file. A reopen that found damagediff --git a/crates/pa-daemon/src/session_store/read.rs b/crates/pa-daemon/src/session_store/read.rsindex 6e2e4032b0..246513d7ba 100644--- a/crates/pa-daemon/src/session_store/read.rs+++ b/crates/pa-daemon/src/session_store/read.rs@@ -156,6 +156,7 @@ impl SessionFile {leaf_id: None,window: None,lease: None,+ trace_upload: None,anthropic_warning_shown: false,skipped_lines: 0,};@@ -214,6 +215,7 @@ impl SessionFile {leaf_id: None,window: None,lease: None,+ trace_upload: None,anthropic_warning_shown,skipped_lines: 0,};@@ -281,6 +283,7 @@ impl SessionFile {}full.leaf_id.clone_from(&self.leaf_id);full.lease.clone_from(&self.lease);+ full.trace_upload.clone_from(&self.trace_upload);// The merged store's leaf is the window's leaf: re-derive the gate// from the merged ACTIVE branch — the full open's own-leaf answer// can disagree, a post-snapshot marker must hydrate, and an@@ -309,6 +312,7 @@ impl SessionFile {leaf_id: None,window: None,lease: None,+ trace_upload: None,anthropic_warning_shown: false,skipped_lines: 0,}diff --git a/crates/pa-daemon/src/session_store/tests.rs b/crates/pa-daemon/src/session_store/tests.rsindex e719f3d53b..5a971e88e4 100644--- a/crates/pa-daemon/src/session_store/tests.rs+++ b/crates/pa-daemon/src/session_store/tests.rs@@ -1262,3 +1262,27 @@ fn session_header_line_leads_with_the_type_tag() {let line = serde_json::to_string(&session_header_line(&header)).unwrap();assert!(line.starts_with("{\"type\":\"session\",\"version\":3,\"id\":\"abc\""));}++#[cfg(unix)] // Windows uses the full reader rather than windowed hydration.+#[test]+fn hydrating_full_history_preserves_the_trace_controller() {+ let dir = temp_dir();+ let path = dir.join("trace-hydration.jsonl");+ let mut original = SessionFile::create(dir.to_str().unwrap(), None, 0);+ original.append_message(&json!({"role":"user", "content":"synthetic", "timestamp":1u64}));+ original.set_path(path.clone());+ original.rewrite().unwrap();+ let mut windowed = SessionFile::open_windowed(&path).unwrap();+ assert!(windowed.window.is_some());+ let (_, consent) = pa_core::agent_traces::ContinuousTraceUpload::load_settings(&dir, &dir);+ let controller =+ pa_core::agent_traces::ContinuousTraceUpload::install(&dir, &dir, Some(&path), consent)+ .unwrap();+ windowed.trace_upload = Some(controller.clone());+ windowed.ensure_full_history().unwrap();+ assert!(windowed.window.is_none());+ assert!(std::sync::Arc::ptr_eq(+ windowed.trace_upload.as_ref().unwrap(),+ &controller+ ));+}diff --git a/crates/pa-daemon/src/session_store/write.rs b/crates/pa-daemon/src/session_store/write.rsindex ffb580de94..9692197a3a 100644--- a/crates/pa-daemon/src/session_store/write.rs+++ b/crates/pa-daemon/src/session_store/write.rs@@ -40,6 +40,9 @@ impl SessionFile {pa_core::session::window::AppendOwnership::Unleased,)?,}+ if let Some(traces) = &self.trace_upload {+ traces.persisted(&self.path);+ }Ok(())}@@ -128,6 +131,9 @@ impl SessionFile {}pa_core::platform::rename_onto(&temp, path).with_context(|| format!("persist {}", path.display()))?;+ if let Some(traces) = &self.trace_upload {+ traces.persisted(&self.path);+ }Ok(())}@@ -216,6 +222,9 @@ impl SessionFile {// The line is in the file now: index it so the in-memory leaf matches what a// reload sees.self.push_index(entry);+ if let Some(traces) = &self.trace_upload {+ traces.persisted(&self.path);+ }} else {// The rewrite path serializes the whole index, so the entry must be// indexed first; a failed rewrite rolls the index back. The livediff --git a/crates/pa-daemon/src/session_tree.rs b/crates/pa-daemon/src/session_tree.rsindex f631008f53..a74568b828 100644--- a/crates/pa-daemon/src/session_tree.rs+++ b/crates/pa-daemon/src/session_tree.rs@@ -245,6 +245,10 @@ impl SessionFile {// Same directory as the source session, like TS `createUniqueSessionFileTarget`.let file = session_dir.join(session_file_name(forked.session_id()));forked.set_path(file);+ forked.trace_upload = self+ .trace_upload+ .as_ref()+ .and_then(|traces| traces.forked(&forked.path));if let Some(lease) = &self.lease {forked.lease = Some(lease.acquire_target(&forked.path)?);}diff --git a/crates/pa-daemon/src/trace_test_env.rs b/crates/pa-daemon/src/trace_test_env.rsnew file mode 100644index 0000000000..bc619a3a5b--- /dev/null+++ b/crates/pa-daemon/src/trace_test_env.rs@@ -0,0 +1,45 @@+//! Test-only: the trace uploader resolves credentials from process env+//! (`PRIME_AGENT_TRACES_API_KEY`, then `PRIME_API_KEY`), so daemon tests+//! that install consent-enabled controllers must clear both and serialize+//! with each other while the values stay stable.++use std::sync::MutexGuard;++pub(crate) struct TraceEnv {+ // Held for the guard's lifetime; the lock's value is never read.+ #[allow(dead_code)]+ lock: MutexGuard<'static, ()>,+ saved: [(&'static str, Option<std::ffi::OsString>); 2],+}++pub(crate) fn lock_env() -> TraceEnv {+ let lock = env_lock();+ let saved = [+ (+ "PRIME_AGENT_TRACES_API_KEY",+ std::env::var_os("PRIME_AGENT_TRACES_API_KEY"),+ ),+ ("PRIME_API_KEY", std::env::var_os("PRIME_API_KEY")),+ ];+ std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");+ std::env::remove_var("PRIME_API_KEY");+ TraceEnv { lock, saved }+}++impl Drop for TraceEnv {+ fn drop(&mut self) {+ for (name, value) in &self.saved {+ if let Some(value) = value {+ std::env::set_var(name, value);+ } else {+ std::env::remove_var(name);+ }+ }+ }+}++fn env_lock() -> MutexGuard<'static, ()> {+ static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());+ LOCK.lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner)+}diff --git a/crates/pa-daemon/src/worker/create.rs b/crates/pa-daemon/src/worker/create.rsindex 6649e716e6..44f4a1305c 100644--- a/crates/pa-daemon/src/worker/create.rs+++ b/crates/pa-daemon/src/worker/create.rs@@ -61,7 +61,9 @@ impl Worker {return response_failure(None,"create",- &format!("Invalid thinking level \"{level}\". Valid values: off, minimal, low, medium, high, xhigh, max"),+ &format!(+ "Invalid thinking level \"{level}\". Valid values: off, minimal, low, medium, high, xhigh, max"+ ),None,);}@@ -197,6 +199,18 @@ impl Worker {// The fresh arms defer their creation prefix to after the startup scope// registers, so a fresh `--models` session persists the scoped startup pick.let mut fresh_prefix = FreshPrefixPlan::None;+ let (settings, trace_consent) = pa_core::agent_traces::ContinuousTraceUpload::load_settings(+ std::path::Path::new(&cwd),+ &self.config.agent_dir,+ );+ let traces = |path: &std::path::Path| {+ pa_core::agent_traces::ContinuousTraceUpload::install(+ std::path::Path::new(&cwd),+ &self.config.agent_dir,+ Some(path),+ trace_consent.clone(),+ )+ };let mut store = match (&session_path, no_session) {(Some(path), false) if path.exists() => {let loaded = {@@ -215,6 +229,7 @@ impl Worker {};match loaded {Ok(mut opened) => {+ opened.trace_upload = traces(&opened.path);opened_existing_session = true;if opened.skipped_lines > 0 {// The rows stay on disk (the append-only@@ -298,6 +313,7 @@ impl Worker {rlm_depth.unwrap_or(0),);created.set_path(path.clone());+ created.trace_upload = traces(&created.path);let acquired = {let path = path.clone();let agent_dir = self.config.agent_dir.clone();@@ -336,6 +352,7 @@ impl Worker {);let path = session_dir.join(session_file_name(created.session_id()));created.set_path(path.clone());+ created.trace_upload = traces(&created.path);let acquired = {let path = path.clone();let agent_dir = self.config.agent_dir.clone();@@ -362,7 +379,7 @@ impl Worker {"create","Session cannot be both no-session and session-pathed",None,- )+ );}};@@ -468,7 +485,6 @@ impl Worker {// The session's settings-seeded switches: a restarted session re-seeds// its auto-compaction flag from the persisted `compaction.enabled`.let (service_tier, steering_mode, follow_up_mode, auto_compaction_enabled) = {- let settings = pa_core::settings::SettingsManager::create(&cwd, &self.config.agent_dir);let queue_mode = |mode: pa_core::settings::QueueModeSetting| -> String {match mode {pa_core::settings::QueueModeSetting::All => "all".to_string(),