Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

restore continuous background trace uploads (#3415)

kt2a16de9060fd
diff --git a/crates/pa-core/README.md b/crates/pa-core/README.md
index c249f76779..bdca2643cb 100644
--- a/crates/pa-core/README.md
+++ b/crates/pa-core/README.md
@@ -78,3 +78,30 @@ No provider HTTP (pa-ai), no loop policy (pa-agent), no daemon supervision (pa-d
## Depends on
pa-types, pa-ai, pa-agent (one-way).
+
+
+Automatic trace delivery (`agent_traces::ContinuousTraceUpload`) owns consent-gated
+persist scheduling and startup outbox recovery alongside the existing manual uploader.
+Hosts reuse `load_settings` to capture consent before their existing settings parse,
+attach `persisted` after successful disk writes, and own the returned installation
+for the session lifetime. Replacements use `rebind` with the effective cwd; forks
+use `forked`. The installation never drains on drop. A detached process-wide runtime
+owns all transcript reads, scans, settings reloads, networking and retries; a maximum
+of 256 registered controllers and four deliveries bounds resident work. Admission
+returns `None` at capacity and logs rejection; hosts attach no hook in that case.
+Retired descriptors free capacity only after background recovery acknowledges them.
+Session replacement transfers its existing slot and cancels the predecessor; forks
+require a distinct slot. Replacement retains a bounded retirement flag until the
+background recovery snapshot acknowledges its exact controller identity. The synchronous
+persist seam checks the cached consent snapshot and performs an exists/create
+pending-marker write; consent metadata reads and reloads stay in the worker after
+the host captures its initial settings generation; it does not fsync the marker (matching TS v0.9.8 process-crash
+recovery rather than promising power-loss durability). A short nonblocking mutation lease prevents cursor/prune races; contention
+records one stable fallback marker per session, which is retained conservatively
+until a future sender has delivered it. Its cost must be measured
+separately; startup/paint parity is not established by scheduler unit tests.
+Active requests monitor consent changes every second on the worker and cancel on
+revocation/errors; bytes already sent cannot be recalled. Recovery belongs to all
+live hosts sharing the agent directory and cancels when the last host disappears.
+Catch-up uses at most three delivery cycles per pending entry, honors Retry-After,
+and releases upload capacity during retry waits; exhaustion retains durable intent.
diff --git a/crates/pa-core/src/agent_traces.rs b/crates/pa-core/src/agent_traces.rs
index fd6a48356c..d4016a3c47 100644
--- a/crates/pa-core/src/agent_traces.rs
+++ b/crates/pa-core/src/agent_traces.rs
@@ -25,6 +25,9 @@ pub use upload_all::{
TraceUploadAllOptions,
};
+mod continuous;
+pub use continuous::{ContinuousTraceUpload, TraceConsentSnapshot};
+
mod upload;
use upload::perform_agent_trace_upload;
pub use upload::{upload_trace_file, TraceUploadOptions};
@@ -287,20 +290,35 @@ impl TraceUploadSignature {
// Session header + context
fn read_trace_session_header(path: &Path) -> Option<pa_types::session::SessionHeader> {
+ read_trace_session_header_checked(path).ok().flatten()
+}
+
+fn read_trace_session_header_checked(
+ path: &Path,
+) -> std::io::Result<Option<pa_types::session::SessionHeader>> {
use std::io::BufRead;
- let file = std::fs::File::open(path).ok()?;
+ let file = std::fs::File::open(path)?;
let mut first_line = String::new();
- std::io::BufReader::new(file)
+ // Corrupt headers must not allocate an entire unbounded transcript line.
+ if let Err(error) = std::io::Read::take(std::io::BufReader::new(file), 256 * 1024 + 1)
.read_line(&mut first_line)
- .ok()?;
- if first_line.trim().is_empty() {
- return None;
+ {
+ return if error.kind() == std::io::ErrorKind::InvalidData {
+ Ok(None)
+ } else {
+ Err(error)
+ };
+ }
+ if first_line.len() > 256 * 1024 || first_line.trim().is_empty() {
+ return Ok(None);
}
- let value: Value = serde_json::from_str(first_line.trim()).ok()?;
+ let Ok(value) = serde_json::from_str::<Value>(first_line.trim()) else {
+ return Ok(None);
+ };
if !is_trace_session_header(&value) {
- return None;
+ return Ok(None);
}
- serde_json::from_value(value).ok()
+ Ok(serde_json::from_value(value).ok())
}
fn is_trace_session_header(value: &Value) -> bool {
@@ -510,8 +528,17 @@ fn read_agent_trace_outbox_entry(
agent_dir: &Path,
session_file: &Path,
) -> Option<TraceUploadSignature> {
+ use std::io::Read;
let entry_path = agent_trace_outbox_entry_path(agent_dir, session_file);
- let raw = std::fs::read_to_string(entry_path).ok()?;
+ let mut raw = String::new();
+ std::fs::File::open(entry_path)
+ .ok()?
+ .take(64 * 1024 + 1)
+ .read_to_string(&mut raw)
+ .ok()?;
+ if raw.len() > 64 * 1024 {
+ return None;
+ }
let (recorded_file, uploaded) = parse_outbox_entry(&raw)?;
if recorded_file != session_file.to_string_lossy() {
return None;
@@ -523,6 +550,25 @@ fn signature_equals(recorded: Option<TraceUploadSignature>, current: TraceUpload
recorded.is_some_and(|recorded| recorded == current)
}
+// Separate short mutation leases from delivery leases (which cover network
+// waits). Persist callers only try-lock; cursor/prune work stays in background.
+fn outbox_mutation_lock(entry: &Path) -> std::io::Result<std::fs::File> {
+ let lock = entry.with_extension("mutation-lock");
+ let mut options = std::fs::OpenOptions::new();
+ options.create(true).read(true).write(true).truncate(false);
+ crate::platform::perms::set_private_mode(&mut options);
+ match options.open(&lock) {
+ Ok(file) => Ok(file),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
+ if let Some(parent) = lock.parent() {
+ std::fs::create_dir_all(parent)?;
+ }
+ options.open(lock)
+ }
+ Err(error) => Err(error),
+ }
+}
+
fn record_agent_trace_outbox_upload(
agent_dir: &Path,
session_file: &Path,
@@ -530,6 +576,8 @@ fn record_agent_trace_outbox_upload(
) -> std::io::Result<()> {
std::fs::create_dir_all(agent_trace_outbox_dir(agent_dir))?;
let entry_path = agent_trace_outbox_entry_path(agent_dir, session_file);
+ let mutation = outbox_mutation_lock(&entry_path)?;
+ crate::platform::lock_exclusive(&mutation)?;
let temp = entry_path.with_extension(format!(
"{}.{}.tmp",
std::process::id(),
diff --git a/crates/pa-core/src/agent_traces/continuous.rs b/crates/pa-core/src/agent_traces/continuous.rs
new file mode 100644
index 0000000000..4f0723fa6e
--- /dev/null
+++ b/crates/pa-core/src/agent_traces/continuous.rs
@@ -0,0 +1,1057 @@
+//! Consent-gated automatic delivery. The detached process-wide runtime owns all
+//! reads, recovery and delivery; hosts only record bounded intent and wake it.
+use super::*;
+use std::io::Read;
+use std::sync::{Mutex, OnceLock, Weak};
+use std::time::Duration;
+use tokio::time::Instant;
+
+const DEBOUNCE: Duration = Duration::from_secs(1);
+const MIN_INTERVAL: Duration = Duration::from_secs(60);
+const MAX_CONTROLLERS: usize = 256;
+
+#[derive(Default, Debug)]
+struct Schedule {
+ due: Option<Instant>,
+ last_start: Option<Instant>,
+ not_before: Option<Instant>,
+ generation: u64,
+}
+
+impl Schedule {
+ fn persist(&mut self, now: Instant) {
+ self.generation = self.generation.wrapping_add(1);
+ self.due = Some(self.deadline(now));
+ }
+
+ fn deadline(&self, now: Instant) -> Instant {
+ let mut due = now + DEBOUNCE;
+ if let Some(start) = self.last_start {
+ due = due.max(start + MIN_INTERVAL);
+ }
+ if let Some(not_before) = self.not_before {
+ due = due.max(not_before);
+ }
+ due
+ }
+
+ fn start(&mut self, now: Instant) -> u64 {
+ self.due = None;
+ self.last_start = Some(now);
+ self.generation
+ }
+
+ fn settle(&mut self, now: Instant, started_generation: u64, result: &TraceUploadResult) {
+ let retry = matches!(
+ result,
+ TraceUploadResult::Disabled | TraceUploadResult::MissingCredentials
+ ) || matches!(result, TraceUploadResult::Failed { status_code, .. }
+ if status_code.is_none_or(|status| status == 429 || RETRIABLE_HTTP_STATUSES.contains(&status)));
+ if let TraceUploadResult::Failed {
+ retry_after_ms: Some(ms),
+ ..
+ } = result
+ {
+ self.not_before = Some(now + Duration::from_millis(*ms));
+ }
+ if retry || self.generation != started_generation {
+ self.due = Some(self.deadline(now));
+ }
+ }
+}
+
+// The generation captures a snapshot so a poisoned metadata read fails closed
+// at persist time; a changed settings generation only re-arms delivery, and
+// parsing/reloading (the consent verdict) stays in the worker.
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct ConsentGeneration(Vec<Result<Option<(u64, SystemTime)>, std::io::ErrorKind>>);
+
+impl ConsentGeneration {
+ fn read(cwd: &Path, agent_dir: &Path) -> Self {
+ Self(
+ [
+ agent_dir.join("settings.json"),
+ cwd.join(crate::settings::storage::CONFIG_DIR_NAME)
+ .join("settings.json"),
+ ]
+ .iter()
+ .map(|path| match std::fs::metadata(path) {
+ Ok(m) => m
+ .modified()
+ .map(|mtime| Some((m.len(), mtime)))
+ .map_err(|e| e.kind()),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+ Err(error) => Err(error.kind()),
+ })
+ .collect(),
+ )
+ }
+}
+
+/// Consent captured around the host's settings load. The generation must be
+/// captured before parsing so a concurrent settings change fails closed.
+#[derive(Clone)]
+pub struct TraceConsentSnapshot {
+ enabled: bool,
+ generation: ConsentGeneration,
+}
+
+/// Session-owned installation. Dropping the last host reference cancels delivery;
+/// no host shutdown awaits this worker or its retry timers.
+pub struct ContinuousTraceUpload {
+ cwd: PathBuf,
+ agent_dir: Arc<PathBuf>,
+ consent: Mutex<(bool, ConsentGeneration)>,
+ // The path is passed by the writer: forks/rebindings cannot upload a stale path.
+ pending: Mutex<Option<(PathBuf, Schedule)>>,
+ wake: Arc<tokio::sync::Notify>,
+ cancel: TraceUploadCancel,
+ started: AtomicBool,
+}
+
+impl std::fmt::Debug for ContinuousTraceUpload {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("ContinuousTraceUpload")
+ .finish_non_exhaustive()
+ }
+}
+
+impl Drop for ContinuousTraceUpload {
+ fn drop(&mut self) {
+ self.cancel.cancel();
+ self.wake.notify_one();
+ }
+}
+
+impl ContinuousTraceUpload {
+ /// Reuse this settings load for the host's other settings. Only two bounded
+ /// metadata reads are added; parsing is the existing host configuration load.
+ #[must_use]
+ pub fn load_settings(
+ cwd: &Path,
+ agent_dir: &Path,
+ ) -> (crate::settings::SettingsManager, TraceConsentSnapshot) {
+ let generation = ConsentGeneration::read(cwd, agent_dir);
+ let settings = crate::settings::SettingsManager::create(cwd, agent_dir);
+ let consent = TraceConsentSnapshot {
+ enabled: settings.errors().is_empty() && settings.get_agent_traces_enabled(),
+ generation,
+ };
+ (settings, consent)
+ }
+
+ fn consent_for(&self, cwd: &Path) -> TraceConsentSnapshot {
+ let Ok(consent) = self.consent.lock() else {
+ return TraceConsentSnapshot {
+ enabled: false,
+ generation: ConsentGeneration::read(cwd, &self.agent_dir),
+ };
+ };
+ TraceConsentSnapshot {
+ enabled: cwd == self.cwd && consent.0,
+ generation: if cwd == self.cwd {
+ consent.1.clone()
+ } else {
+ ConsentGeneration::read(cwd, &self.agent_dir)
+ },
+ }
+ }
+
+ /// Retire this installation and bind its registry slot to a replacement.
+ /// A cwd change starts with consent off until background verification.
+ #[must_use]
+ pub fn rebind(&self, cwd: &Path, path: &Path) -> Option<Arc<Self>> {
+ let controller =
+ Self::unregistered(cwd, &self.agent_dir, Some(path), self.consent_for(cwd));
+ if service().replace(self, &controller) {
+ Some(controller)
+ } else {
+ tracing::warn!("trace replacement rejected: source registration unavailable");
+ None
+ }
+ }
+
+ /// Fork within this installation's cwd, admitting a distinct installation.
+ #[must_use]
+ pub fn forked(&self, path: &Path) -> Option<Arc<Self>> {
+ Self::install(
+ &self.cwd,
+ &self.agent_dir,
+ Some(path),
+ self.consent_for(&self.cwd),
+ )
+ }
+
+ fn unregistered(
+ cwd: &Path,
+ agent_dir: &Path,
+ session_file: Option<&Path>,
+ consent: TraceConsentSnapshot,
+ ) -> Arc<Self> {
+ Arc::new(Self {
+ cwd: cwd.to_path_buf(),
+ agent_dir: Arc::new(agent_dir.to_path_buf()),
+ consent: Mutex::new((consent.enabled, consent.generation)),
+ pending: Mutex::new(session_file.map(|p| (p.to_path_buf(), Schedule::default()))),
+ wake: Arc::new(tokio::sync::Notify::new()),
+ cancel: TraceUploadCancel::new(),
+ started: AtomicBool::new(false),
+ })
+ }
+
+ /// Install without scanning the outbox or waiting for networking. Consent
+ /// is captured around the host's existing settings load.
+ /// Returns `None` when the bounded registry cannot admit this installation.
+ /// No persist hook may be attached to a rejected installation.
+ #[must_use]
+ pub fn install(
+ cwd: &Path,
+ agent_dir: &Path,
+ session_file: Option<&Path>,
+ consent: TraceConsentSnapshot,
+ ) -> Option<Arc<Self>> {
+ let controller = Self::unregistered(cwd, agent_dir, session_file, consent);
+ let service = service();
+ if service.register(&controller) {
+ Some(controller)
+ } else {
+ tracing::warn!("trace installation rejected: registry unavailable or at capacity");
+ None
+ }
+ }
+
+ /// Called only after a successful transcript write. This performs no
+ /// transcript read, settings parse, directory scan or network. Only the small
+ /// pending record is serialized synchronously.
+ pub fn persisted(&self, session_file: &Path) {
+ if session_file.as_os_str().is_empty() || self.cancel.is_cancelled() {
+ return;
+ }
+ let Ok(consent) = self.consent.lock() else {
+ return;
+ };
+ if !consent.0 || consent.1 .0.iter().any(Result::is_err) {
+ return;
+ }
+ drop(consent);
+ let began = std::time::Instant::now();
+ if let Err(error) = mark_pending(&self.agent_dir, session_file) {
+ tracing::warn!(%error, "trace pending marker failed");
+ }
+ tracing::trace!(
+ elapsed_us = began.elapsed().as_micros() as u64,
+ "trace pending marker duration"
+ );
+ let Ok(mut pending) = self.pending.lock() else {
+ return;
+ };
+ let (path, schedule) =
+ pending.get_or_insert_with(|| (session_file.to_path_buf(), Schedule::default()));
+ if path != session_file {
+ *path = session_file.to_path_buf();
+ *schedule = Schedule::default();
+ }
+ schedule.persist(Instant::now());
+ drop(pending);
+ self.wake.notify_one();
+ }
+}
+
+fn mark_pending(agent_dir: &Path, session_file: &Path) -> std::io::Result<()> {
+ let primary = agent_trace_outbox_entry_path(agent_dir, session_file);
+ let mutation = outbox_mutation_lock(&primary)?;
+ // Never wait behind a pruner or cursor writer on the host thread. A stable
+ // fallback marker retains intent even if the primary is about to be pruned.
+ let acquired = crate::platform::try_lock_exclusive(&mutation).is_ok();
+ let entry = if acquired {
+ primary
+ } else {
+ primary.with_extension("pending.json")
+ };
+ if entry.exists() {
+ return Ok(());
+ }
+ std::fs::create_dir_all(agent_trace_outbox_dir(agent_dir))?;
+ // Publish a complete record without replacing a racing successful cursor.
+ let temp = entry.with_extension(format!(
+ "{}.{}.tmp",
+ std::process::id(),
+ uuid::Uuid::new_v4()
+ ));
+ let result = (|| {
+ use std::io::Write;
+ let mut options = std::fs::OpenOptions::new();
+ options.write(true).create_new(true);
+ crate::platform::perms::set_private_mode(&mut options);
+ let mut file = options.open(&temp)?;
+ writeln!(
+ file,
+ "{}",
+ json!({"sessionFile": session_file.to_string_lossy()})
+ )?;
+ match std::fs::hard_link(&temp, &entry) {
+ Ok(()) => Ok(()),
+ Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Ok(()),
+ Err(error) => Err(error),
+ }
+ })();
+ let _ = std::fs::remove_file(temp);
+ result
+}
+
+fn prune_entry(entry: &Path, observed: &str, missing_session: Option<&Path>) {
+ // Fallback markers are deliberately retained: another process can publish
+ // intent there without waiting while the primary's short lease is held.
+ if entry
+ .file_name()
+ .is_some_and(|name| name.to_string_lossy().ends_with(".pending.json"))
+ {
+ return;
+ }
+ let Ok(mutation) = outbox_mutation_lock(entry) else {
+ return;
+ };
+ if crate::platform::try_lock_exclusive(&mutation).is_err() {
+ return;
+ }
+ let mut current = String::new();
+ let Ok(file) = std::fs::File::open(entry) else {
+ return;
+ };
+ if file
+ .take(64 * 1024 + 1)
+ .read_to_string(&mut current)
+ .is_err()
+ || current != observed
+ {
+ return;
+ }
+ if let Some(session) = missing_session {
+ match std::fs::metadata(session) {
+ Ok(meta) if meta.is_file() => return,
+ Ok(_) => {}
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+ Err(_) => return,
+ }
+ }
+ if let Err(error) = std::fs::remove_file(entry) {
+ tracing::debug!(%error, "trace stale marker pruning failed");
+ }
+}
+
+fn delivery_lease(agent_dir: &Path, path: &Path) -> Option<std::fs::File> {
+ let lock = agent_trace_outbox_entry_path(agent_dir, path).with_extension("lock");
+ std::fs::create_dir_all(agent_trace_outbox_dir(agent_dir)).ok()?;
+ let mut options = std::fs::OpenOptions::new();
+ options.create(true).read(true).write(true).truncate(false);
+ crate::platform::perms::set_private_mode(&mut options);
+ let file = options.open(lock).ok()?;
+ crate::platform::try_lock_exclusive(&file).ok()?;
+ Some(file)
+}
+
+#[derive(Clone)]
+struct Registration {
+ controller: Weak<ContinuousTraceUpload>,
+ agent_dir: Arc<PathBuf>,
+ retired_predecessor: bool,
+}
+
+impl Registration {
+ fn of(controller: &Arc<ContinuousTraceUpload>) -> Self {
+ Self {
+ controller: Arc::downgrade(controller),
+ agent_dir: controller.agent_dir.clone(),
+ retired_predecessor: false,
+ }
+ }
+}
+
+struct Service {
+ controllers: Mutex<Vec<Registration>>,
+ wake: tokio::sync::Notify,
+}
+
+impl Service {
+ fn register(&self, controller: &Arc<ContinuousTraceUpload>) -> bool {
+ let Ok(mut registrations) = self.controllers.lock() else {
+ return false;
+ };
+ // Only the background service acknowledges dead descriptors, preserving
+ // retirements between ticks. Rejection never returns a usable controller.
+ if registrations.len() >= MAX_CONTROLLERS {
+ return false;
+ }
+ registrations.push(Registration::of(controller));
+ self.wake.notify_one();
+ true
+ }
+
+ fn replace(
+ &self,
+ source: &ContinuousTraceUpload,
+ controller: &Arc<ContinuousTraceUpload>,
+ ) -> bool {
+ let Ok(mut registrations) = self.controllers.lock() else {
+ return false;
+ };
+ let Some(slot) = registrations
+ .iter_mut()
+ .find(|registration| std::ptr::eq(registration.controller.as_ptr(), source))
+ else {
+ return false;
+ };
+ // Reuse exactly one slot. The flag retains retirement even when neither
+ // predecessor nor replacement survives until the next service tick.
+ *slot = Registration::of(controller);
+ slot.retired_predecessor = true;
+ source.cancel.cancel();
+ source.wake.notify_one();
+ self.wake.notify_one();
+ true
+ }
+}
+
+fn service() -> &'static Service {
+ static SERVICE: OnceLock<Service> = OnceLock::new();
+ SERVICE.get_or_init(|| {
+ let service = Service {
+ controllers: Mutex::new(Vec::new()),
+ wake: tokio::sync::Notify::new(),
+ };
+ // Detached OS threads do not extend process lifetime. The host's Tokio
+ // blocking pool is never used by uploads or outbox recovery.
+ if let Err(error) = std::thread::Builder::new()
+ .name("trace-upload".into())
+ .spawn(|| {
+ let runtime = match tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ {
+ Ok(runtime) => runtime,
+ Err(error) => {
+ tracing::warn!(%error, "trace runtime failed");
+ return;
+ }
+ };
+ runtime.block_on(run_service());
+ })
+ {
+ tracing::warn!(%error, "trace thread failed");
+ }
+ service
+ })
+}
+
+// Recovery phases: 0 pending, 1 running/cooling down, 2 completed.
+struct RecoveryRun {
+ cancel: TraceUploadCancel,
+ state: Arc<std::sync::atomic::AtomicU8>,
+ replay_needed: Arc<AtomicBool>,
+ not_before: Arc<Mutex<Option<Instant>>>,
+ // Weak identities pin allocations, so retirement detection cannot reuse addresses.
+ hosts: Vec<Weak<ContinuousTraceUpload>>,
+}
+
+impl RecoveryRun {
+ fn new(hosts: Vec<Weak<ContinuousTraceUpload>>) -> Self {
+ Self {
+ cancel: TraceUploadCancel::new(),
+ state: Arc::new(std::sync::atomic::AtomicU8::new(0)),
+ replay_needed: Arc::new(AtomicBool::new(false)),
+ not_before: Arc::new(Mutex::new(None)),
+ hosts,
+ }
+ }
+}
+
+type RecoveryRuns = std::collections::HashMap<PathBuf, RecoveryRun>;
+
+fn retain_live_recoveries(
+ recovered: &mut RecoveryRuns,
+ registrations: &[Registration],
+) -> Vec<Weak<ContinuousTraceUpload>> {
+ let mut live: std::collections::HashMap<PathBuf, Vec<Weak<ContinuousTraceUpload>>> =
+ std::collections::HashMap::new();
+ let mut retired_directories = std::collections::HashSet::new();
+ let mut retired = Vec::new();
+ for registration in registrations {
+ if registration.retired_predecessor {
+ retired_directories.insert(registration.agent_dir.as_ref().clone());
+ }
+ if let Some(controller) = registration.controller.upgrade() {
+ live.entry(controller.agent_dir.as_ref().clone())
+ .or_default()
+ .push(registration.controller.clone());
+ } else {
+ retired_directories.insert(registration.agent_dir.as_ref().clone());
+ retired.push(registration.controller.clone());
+ }
+ }
+ recovered.retain(|directory, run| {
+ let Some(hosts) = live.get(directory) else {
+ run.cancel.cancel();
+ return false;
+ };
+ if retired_directories.contains(directory)
+ || run
+ .hosts
+ .iter()
+ .any(|old| !hosts.iter().any(|new| old.ptr_eq(new)))
+ {
+ run.replay_needed.store(true, Ordering::Release);
+ }
+ run.hosts.clone_from(hosts);
+ true
+ });
+ retired
+}
+
+fn acknowledge_retired_registrations(
+ registrations: &mut Vec<Registration>,
+ retired: &[Weak<ContinuousTraceUpload>],
+) {
+ // The retained Weak handles pin allocations until acknowledgement finishes.
+ // Hosts retiring after the snapshot remain registered for the next tick.
+ let identities: std::collections::HashSet<_> =
+ retired.iter().map(|weak| weak.as_ptr() as usize).collect();
+ registrations
+ .retain(|registration| !identities.contains(&(registration.controller.as_ptr() as usize)));
+}
+
+fn acknowledge_replacements(registrations: &mut [Registration], snapshot: &[Registration]) {
+ let identities: std::collections::HashSet<_> = snapshot
+ .iter()
+ .filter(|registration| registration.retired_predecessor)
+ .map(|registration| registration.controller.as_ptr() as usize)
+ .collect();
+ // Snapshot Weak handles pin identity; a later replacement keeps its flag.
+ for registration in registrations {
+ if identities.contains(&(registration.controller.as_ptr() as usize)) {
+ registration.retired_predecessor = false;
+ }
+ }
+}
+
+fn rearm_retired_recovery(run: &RecoveryRun) {
+ if run.replay_needed.load(Ordering::Acquire)
+ && run
+ .state
+ .compare_exchange(2, 1, Ordering::AcqRel, Ordering::Acquire)
+ .is_ok()
+ {
+ let state = run.state.clone();
+ let cancel = run.cancel.clone();
+ let not_before = *run.not_before.lock().unwrap();
+ tokio::spawn(async move {
+ settle_recovery_run(&state, false, not_before, &cancel).await;
+ });
+ }
+}
+
+async fn run_service() {
+ let service = service();
+ let permits = Arc::new(tokio::sync::Semaphore::new(4));
+ // Recovery belongs to all live hosts sharing this directory, not its first host.
+ let mut recovered = RecoveryRuns::new();
+ loop {
+ let registrations = service.controllers.lock().unwrap().clone();
+ let retired = retain_live_recoveries(&mut recovered, &registrations);
+ {
+ let mut current = service.controllers.lock().unwrap();
+ acknowledge_retired_registrations(&mut current, &retired);
+ acknowledge_replacements(&mut current, &registrations);
+ }
+ for registration in registrations {
+ let weak = registration.controller;
+ if let Some(controller) = weak.upgrade() {
+ if controller.consent.lock().unwrap().0
+ && (recovered.contains_key(controller.agent_dir.as_ref())
+ || recovered.len() < MAX_CONTROLLERS)
+ {
+ let run = recovered
+ .entry(controller.agent_dir.as_ref().clone())
+ .or_insert_with(|| RecoveryRun::new(vec![weak.clone()]));
+ rearm_retired_recovery(run);
+ if run
+ .state
+ .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire)
+ .is_ok()
+ {
+ let cwd = controller.cwd.clone();
+ let agent_dir = controller.agent_dir.as_ref().clone();
+ // Clear only on admission: retirement during completion must survive.
+ run.replay_needed.store(false, Ordering::Release);
+ let cancel = run.cancel.clone();
+ let state = run.state.clone();
+ let replay_needed = run.replay_needed.clone();
+ let saved_deadline = run.not_before.clone();
+ let permits = permits.clone();
+ tokio::spawn(async move {
+ let (complete, not_before) = recover_with_backoff(
+ cwd,
+ agent_dir,
+ permits,
+ Arc::new(ReqwestTraceHttp),
+ None,
+ cancel.clone(),
+ None,
+ )
+ .await;
+ *saved_deadline.lock().unwrap() = not_before;
+ let complete = complete && !replay_needed.load(Ordering::Acquire);
+ settle_recovery_run(&state, complete, not_before, &cancel).await;
+ });
+ }
+ }
+ if !controller.started.swap(true, Ordering::AcqRel) {
+ tokio::spawn(run_controller(
+ weak,
+ permits.clone(),
+ Arc::new(ReqwestTraceHttp),
+ None,
+ ));
+ }
+ }
+ }
+ tokio::select! { () = service.wake.notified() => {}, () = tokio::time::sleep(DEBOUNCE) => {} }
+ }
+}
+
+async fn settle_recovery_run(
+ state: &std::sync::atomic::AtomicU8,
+ complete: bool,
+ not_before: Option<Instant>,
+ cancel: &TraceUploadCancel,
+) {
+ if !complete {
+ // Mixed-consent directories may need another background sweep. Keep its
+ // state running during cooldown; the service rechecks consent before rearm.
+ let due = (Instant::now() + MIN_INTERVAL).max(not_before.unwrap_or_else(Instant::now));
+ tokio::select! {
+ () = tokio::time::sleep_until(due) => {},
+ () = cancel.wait() => {},
+ }
+ }
+ state.store(if complete { 2 } else { 0 }, Ordering::Release);
+}
+
+async fn run_controller(
+ weak: Weak<ContinuousTraceUpload>,
+ permits: Arc<tokio::sync::Semaphore>,
+ http: Arc<dyn TraceHttp>,
+ base_url: Option<String>,
+) {
+ if let Some(controller) = weak.upgrade() {
+ let path = controller
+ .pending
+ .lock()
+ .unwrap()
+ .as_ref()
+ .map(|(p, _)| p.clone());
+ if let Some(path) = path {
+ let entry = agent_trace_outbox_entry_path(&controller.agent_dir, &path);
+ let pending_on_disk = entry.is_file() || entry.with_extension("pending.json").is_file();
+ if pending_on_disk {
+ let mut pending = controller.pending.lock().unwrap();
+ if let Some((current, schedule)) = pending.as_mut() {
+ if current == &path {
+ schedule.persist(Instant::now());
+ }
+ }
+ }
+ }
+ }
+ let mut initialized = false;
+ loop {
+ let Some(controller) = weak.upgrade() else {
+ return;
+ };
+ let generation = ConsentGeneration::read(&controller.cwd, &controller.agent_dir);
+ if controller.consent.lock().unwrap().1 != generation || !initialized {
+ let settings = crate::settings::SettingsManager::create(
+ &controller.cwd,
+ controller.agent_dir.as_ref(),
+ );
+ let unchanged =
+ generation == ConsentGeneration::read(&controller.cwd, &controller.agent_dir);
+ *controller.consent.lock().unwrap() = (
+ unchanged && settings.errors().is_empty() && settings.get_agent_traces_enabled(),
+ generation,
+ );
+ initialized = true;
+ }
+ let enabled = controller.consent.lock().unwrap().0;
+ let due = controller
+ .pending
+ .lock()
+ .unwrap()
+ .as_ref()
+ .and_then(|(_, s)| s.due);
+ let wake = controller.wake.clone();
+ let cancel = controller.cancel.clone();
+ // Weak ownership during waits is essential: timers cannot keep a session alive.
+ drop(controller);
+ if !enabled || due.is_none_or(|due| due > Instant::now()) {
+ // Overdue work must not spin or acquire capacity while consent is off.
+ let wait = if enabled {
+ due.map_or(DEBOUNCE, |due| {
+ due.saturating_duration_since(Instant::now()).min(DEBOUNCE)
+ })
+ } else {
+ DEBOUNCE
+ };
+ tokio::select! { () = wake.notified() => {}, () = tokio::time::sleep(wait) => {}, () = cancel.wait() => return }
+ continue;
+ }
+ let permit = tokio::select! { p = permits.clone().acquire_owned() => p.unwrap(), () = cancel.wait() => return };
+ let Some(controller) = weak.upgrade() else {
+ return;
+ };
+ let path = controller
+ .pending
+ .lock()
+ .unwrap()
+ .as_ref()
+ .map(|(path, _)| path.clone());
+ let Some(path) = path else {
+ continue;
+ };
+ let Some(_delivery_lease) = delivery_lease(&controller.agent_dir, &path) else {
+ // A busy lease must not hold delivery capacity while waiting.
+ drop(permit);
+ drop(controller);
+ tokio::select! { () = tokio::time::sleep(DEBOUNCE) => {}, () = cancel.wait() => return }
+ continue;
+ };
+ let leased_path = path;
+ let (path, generation) = {
+ let mut pending = controller.pending.lock().unwrap();
+ let Some((path, schedule)) = pending.as_mut() else {
+ continue;
+ };
+ if path != &leased_path || schedule.due.is_none_or(|due| due > Instant::now()) {
+ continue;
+ }
+ (path.clone(), schedule.start(Instant::now()))
+ };
+ let cwd = controller.cwd.clone();
+ let agent_dir = controller.agent_dir.clone();
+ drop(controller);
+ let result = deliver_with_consent(
+ &TraceUploadOptions {
+ session_file: Some(&path),
+ cwd: &cwd,
+ agent_dir: &agent_dir,
+ require_enabled: true,
+ reload_config: true,
+ base_url: base_url.as_deref(),
+ http: http.as_ref(),
+ request_timeout_ms: DEFAULT_REQUEST_TIMEOUT_MS,
+ cancel: Some(&cancel),
+ on_upload_delay: None,
+ },
+ None,
+ )
+ .await;
+ log_agent_trace_outcome(&agent_dir, Some(&path), &result);
+ drop(permit);
+ if let Some(controller) = weak.upgrade() {
+ let mut pending = controller.pending.lock().unwrap();
+ if let Some((current, schedule)) = pending.as_mut() {
+ if current == &path {
+ schedule.settle(Instant::now(), generation, &result);
+ }
+ }
+ }
+ }
+}
+
+async fn deliver_with_consent(
+ options: &TraceUploadOptions<'_>,
+ gate: Option<&TraceRequestGate>,
+) -> TraceUploadResult {
+ let mut initial = ConsentGeneration::read(options.cwd, options.agent_dir);
+ let request_cancel = TraceUploadCancel::new();
+ let guarded = TraceUploadOptions {
+ cancel: Some(&request_cancel),
+ on_upload_delay: options.on_upload_delay.clone(),
+ ..*options
+ };
+ let request = perform_agent_trace_upload(&guarded, gate);
+ tokio::pin!(request);
+ loop {
+ tokio::select! {
+ result = &mut request => return result,
+ () = async { if let Some(cancel) = options.cancel { cancel.wait().await; } else { std::future::pending::<()>().await; } } => {
+ request_cancel.cancel();
+ return request.await;
+ }
+ () = tokio::time::sleep(DEBOUNCE) => {
+ let current = ConsentGeneration::read(options.cwd, options.agent_dir);
+ if initial != current {
+ let settings = crate::settings::SettingsManager::create(options.cwd, options.agent_dir);
+ if settings.errors().is_empty() && settings.get_agent_traces_enabled()
+ && current == ConsentGeneration::read(options.cwd, options.agent_dir) {
+ initial = current;
+ } else {
+ request_cancel.cancel();
+ let _ = request.await;
+ return TraceUploadResult::Disabled;
+ }
+ }
+ }
+ }
+ }
+}
+
+async fn recover_with_backoff(
+ cwd: PathBuf,
+ agent_dir: PathBuf,
+ permits: Arc<tokio::sync::Semaphore>,
+ http: Arc<dyn TraceHttp>,
+ base_url: Option<String>,
+ cancel: TraceUploadCancel,
+ live_controllers: Option<Vec<Weak<ContinuousTraceUpload>>>,
+) -> (bool, Option<Instant>) {
+ recover_with_io(
+ cwd,
+ agent_dir,
+ permits,
+ http,
+ base_url,
+ cancel,
+ live_controllers,
+ |_, _| Ok(()),
+ )
+ .await
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+enum RecoveryRead {
+ Directory,
+ Iteration,
+ RecordMetadata,
+ Record,
+ SessionMetadata,
+ Header,
+}
+
+// Keep fault injection at the I/O boundary so deterministic recovery tests can
+// exercise transient failures without changing filesystem permissions or users.
+#[allow(clippy::too_many_arguments)]
+async fn recover_with_io(
+ cwd: PathBuf,
+ agent_dir: PathBuf,
+ permits: Arc<tokio::sync::Semaphore>,
+ http: Arc<dyn TraceHttp>,
+ base_url: Option<String>,
+ cancel: TraceUploadCancel,
+ live_controllers: Option<Vec<Weak<ContinuousTraceUpload>>>,
+ before_read: impl Fn(RecoveryRead, &Path) -> std::io::Result<()> + Send + Sync,
+) -> (bool, Option<Instant>) {
+ let settings = crate::settings::SettingsManager::create(&cwd, &agent_dir);
+ if !settings.errors().is_empty() || !settings.get_agent_traces_enabled() {
+ return (false, None);
+ }
+ // Workers share an agent directory: only one startup sweep may deliver it
+ // at a time. OS locks release on crashes without stale-directory retries.
+ let Some(_recovery_lease) = delivery_lease(&agent_dir, &agent_dir.join("catch-up")) else {
+ return (false, None);
+ };
+ let outbox = agent_trace_outbox_dir(&agent_dir);
+ let read_directory = async {
+ before_read(RecoveryRead::Directory, &outbox)?;
+ tokio::fs::read_dir(&outbox).await
+ };
+ let mut entries = match read_directory.await {
+ Ok(entries) => entries,
+ Err(error) => return (error.kind() == std::io::ErrorKind::NotFound, None),
+ };
+ let gate = TraceRequestGate::new();
+ let mut complete = true;
+ let mut not_before: Option<Instant> = None;
+ loop {
+ let next = async {
+ before_read(RecoveryRead::Iteration, &outbox)?;
+ entries.next_entry().await
+ };
+ let entry = match next.await {
+ Ok(Some(entry)) => entry,
+ Ok(None) => break,
+ Err(error) => {
+ // A transient directory-iteration failure must not let the
+ // sweep complete: unexamined markers stay eligible and the
+ // bounded cooldown re-arms another sweep.
+ tracing::warn!(%error, "trace outbox iteration failed");
+ complete = false;
+ break;
+ }
+ };
+ if cancel.is_cancelled() {
+ return (false, None);
+ }
+ if entry.path().extension().is_none_or(|ext| ext != "json") {
+ continue;
+ }
+ // Outbox records contain only a path/cursor; bound corrupt record reads.
+ let record_metadata = async {
+ before_read(RecoveryRead::RecordMetadata, &entry.path())?;
+ entry.metadata().await
+ };
+ match record_metadata.await {
+ Ok(metadata) if metadata.len() > 64 * 1024 => continue,
+ Ok(_) => {}
+ Err(_) => {
+ complete = false;
+ continue;
+ }
+ }
+ let read_entry = async {
+ use tokio::io::AsyncReadExt;
+ before_read(RecoveryRead::Record, &entry.path())?;
+ let file = tokio::fs::File::open(entry.path()).await?;
+ let mut raw = String::new();
+ file.take(64 * 1024 + 1).read_to_string(&mut raw).await?;
+ Ok::<_, std::io::Error>(raw)
+ };
+ let raw = match read_entry.await {
+ Ok(raw) => raw,
+ Err(error) => {
+ // Invalid UTF-8 is terminal data, not a transient filesystem fault.
+ complete &= error.kind() == std::io::ErrorKind::InvalidData;
+ continue;
+ }
+ };
+ if raw.len() > 64 * 1024 {
+ continue;
+ }
+ let Ok(value) = serde_json::from_str::<Value>(&raw) else {
+ prune_entry(&entry.path(), &raw, None);
+ continue;
+ };
+ if value.get("kind").is_some() {
+ continue;
+ }
+ let Some(path) = value
+ .get("sessionFile")
+ .and_then(Value::as_str)
+ .map(PathBuf::from)
+ else {
+ prune_entry(&entry.path(), &raw, None);
+ continue;
+ };
+ // Fulfilled cursor entries owe no bytes even when their owner opted out.
+ if TraceUploadSignature::of(&path).is_some_and(|signature| {
+ read_agent_trace_outbox_entry(&agent_dir, &path) == Some(signature)
+ }) {
+ continue;
+ }
+ let registered = live_controllers.clone().unwrap_or_else(|| {
+ service()
+ .controllers
+ .lock()
+ .unwrap()
+ .iter()
+ .map(|registration| registration.controller.clone())
+ .collect()
+ });
+ let live = registered.iter().filter_map(Weak::upgrade).any(|c| {
+ c.pending
+ .lock()
+ .unwrap()
+ .as_ref()
+ .is_some_and(|(p, _)| p == &path)
+ });
+ if live {
+ complete = false;
+ continue;
+ }
+ let session_metadata = async {
+ before_read(RecoveryRead::SessionMetadata, &path)?;
+ tokio::fs::metadata(&path).await
+ };
+ match session_metadata.await {
+ Ok(meta) if meta.is_file() => {}
+ Ok(_) => {
+ prune_entry(&entry.path(), &raw, Some(&path));
+ continue;
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
+ prune_entry(&entry.path(), &raw, Some(&path));
+ continue;
+ }
+ // A transient metadata error must not let the sweep complete.
+ Err(_) => {
+ complete = false;
+ continue;
+ }
+ }
+ let Some(_delivery_lease) = delivery_lease(&agent_dir, &path) else {
+ // Another process owns this delivery; a skipped marker must not
+ // complete the sweep or it strands when that process fails.
+ complete = false;
+ continue;
+ };
+ let header = before_read(RecoveryRead::Header, &path)
+ .and_then(|()| read_trace_session_header_checked(&path));
+ let header = match header {
+ Ok(Some(header)) => header,
+ Ok(None) => continue,
+ Err(_) => {
+ complete = false;
+ continue;
+ }
+ };
+ let session_cwd = PathBuf::from(&header.cwd);
+ let options = TraceUploadOptions {
+ session_file: Some(&path),
+ cwd: &session_cwd,
+ agent_dir: &agent_dir,
+ require_enabled: true,
+ reload_config: true,
+ base_url: base_url.as_deref(),
+ http: http.as_ref(),
+ request_timeout_ms: DEFAULT_REQUEST_TIMEOUT_MS,
+ cancel: Some(&cancel),
+ on_upload_delay: None,
+ };
+ // Bound catch-up retries per entry; exhaustion leaves its durable marker.
+ let mut schedule = Schedule::default();
+ for attempt in 0..3 {
+ let permit = tokio::select! { p = permits.clone().acquire_owned() => p.unwrap(), () = cancel.wait() => return (false, None) };
+ let generation = schedule.start(Instant::now());
+ let result = deliver_with_consent(&options, Some(&gate)).await;
+ log_agent_trace_outcome(&agent_dir, Some(&path), &result);
+ drop(permit);
+ if matches!(
+ result,
+ TraceUploadResult::Disabled | TraceUploadResult::MissingCredentials
+ ) {
+ // A revoked project must not block other opted-in projects, and
+ // missing credentials may arrive later. Retain the marker and
+ // rearm the shared sweep at a bounded rate.
+ complete = false;
+ break;
+ }
+ schedule.settle(Instant::now(), generation, &result);
+ if let Some(deadline) = schedule.not_before {
+ not_before = Some(not_before.map_or(deadline, |old| old.max(deadline)));
+ }
+ let Some(due) = schedule.due else {
+ break;
+ };
+ if attempt == 2 {
+ // Exhausted per-entry retries with work still due: the durable
+ // marker is retained, so the sweep must rearm, not complete.
+ complete = false;
+ break;
+ }
+ tokio::select! { () = tokio::time::sleep_until(due) => {}, () = cancel.wait() => return (false, None) }
+ }
+ }
+ (complete && !cancel.is_cancelled(), not_before)
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/crates/pa-core/src/agent_traces/continuous/tests.rs b/crates/pa-core/src/agent_traces/continuous/tests.rs
new file mode 100644
index 0000000000..9260ddbd18
--- /dev/null
+++ b/crates/pa-core/src/agent_traces/continuous/tests.rs
@@ -0,0 +1,1536 @@
+use super::*;
+use crate::agent_traces::tests::{
+ clear_trace_credentials, env_lock, response, Fixture, ScriptedTraceHttp,
+};
+use std::io::Write;
+
+async fn recover(
+ cwd: PathBuf,
+ agent_dir: PathBuf,
+ permits: Arc<tokio::sync::Semaphore>,
+ http: Arc<dyn TraceHttp>,
+ base_url: Option<String>,
+ cancel: TraceUploadCancel,
+) -> bool {
+ recover_with_backoff(cwd, agent_dir, permits, http, base_url, cancel, None)
+ .await
+ .0
+}
+
+fn controller(fixture: &Fixture, path: &Path, enabled: bool) -> Arc<ContinuousTraceUpload> {
+ Arc::new(ContinuousTraceUpload {
+ cwd: fixture.cwd.clone(),
+ agent_dir: Arc::new(fixture.agent_dir.clone()),
+ consent: Mutex::new((
+ enabled,
+ ConsentGeneration::read(&fixture.cwd, &fixture.agent_dir),
+ )),
+ pending: Mutex::new(Some((path.to_path_buf(), Schedule::default()))),
+ wake: Arc::new(tokio::sync::Notify::new()),
+ cancel: TraceUploadCancel::new(),
+ started: AtomicBool::new(false),
+ })
+}
+
+#[test]
+fn debounce_throttle_and_writes_in_flight_coalesce() {
+ let now = Instant::now();
+ let mut s = Schedule::default();
+ s.persist(now);
+ s.persist(now + Duration::from_millis(500));
+ assert_eq!(s.due, Some(now + Duration::from_millis(1500)));
+ let generation = s.start(now + Duration::from_millis(1500));
+ s.persist(now + Duration::from_secs(2));
+ s.settle(
+ now + Duration::from_secs(3),
+ generation,
+ &TraceUploadResult::Unchanged,
+ );
+ assert_eq!(s.due, Some(now + Duration::from_millis(61500)));
+ let generation = s.start(s.due.unwrap());
+ s.settle(
+ now + Duration::from_secs(62),
+ generation,
+ &TraceUploadResult::Unchanged,
+ );
+ assert_eq!(s.due, None);
+}
+
+#[test]
+fn rate_limit_reschedule_honors_retry_after_and_terminal_failure_stops() {
+ let now = Instant::now();
+ let mut s = Schedule::default();
+ let generation = s.start(now);
+ s.settle(
+ now + Duration::from_secs(1),
+ generation,
+ &TraceUploadResult::Failed {
+ status_code: Some(429),
+ message: "synthetic".into(),
+ retry_after_ms: Some(120_000),
+ },
+ );
+ assert_eq!(s.due, Some(now + Duration::from_secs(121)));
+ let generation = s.start(s.due.unwrap());
+ s.settle(
+ now + Duration::from_secs(122),
+ generation,
+ &TraceUploadResult::Failed {
+ status_code: Some(403),
+ message: "synthetic".into(),
+ retry_after_ms: None,
+ },
+ );
+ assert_eq!(s.due, None);
+}
+
+#[test]
+fn consent_off_never_registers_or_schedules_retroactively() {
+ let fixture = Fixture::new();
+ let path = fixture.write_session("off.jsonl", "off");
+ let c = controller(&fixture, &path, false);
+ c.persisted(&path);
+ *c.consent.lock().unwrap() = (
+ true,
+ ConsentGeneration::read(&fixture.cwd, &fixture.agent_dir),
+ );
+ assert_eq!(c.pending.lock().unwrap().as_ref().unwrap().1.due, None);
+ assert!(!agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());
+}
+
+#[tokio::test(start_paused = true)]
+async fn external_consent_change_fails_closed_until_background_reload() {
+ let fixture = Fixture::new();
+ let path = fixture.write_session("changed.jsonl", "changed");
+ let c = controller(&fixture, &path, true);
+ std::fs::write(
+ fixture.agent_dir.join("settings.json"),
+ r#"{"agentTraces":{"enabled":false}}"#,
+ )
+ .unwrap();
+ c.persisted(&path);
+ // The durable intent survives the external change; only the worker's
+ // consent re-read decides whether it is ever delivered.
+ assert!(c.pending.lock().unwrap().as_ref().unwrap().1.due.is_some());
+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: Arc::new(tokio::sync::Notify::new()),
+ });
+ let task = tokio::spawn(run_controller(
+ Arc::downgrade(&c),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ ));
+ tokio::time::advance(MIN_INTERVAL * 2).await;
+ tokio::task::yield_now().await;
+ assert!(
+ observed.try_recv().is_err(),
+ "a disabled project must never deliver its retained intent"
+ );
+ drop(c);
+ task.await.unwrap();
+}
+
+#[test]
+fn marker_is_complete_and_preserves_a_success_cursor() {
+ let fixture = Fixture::new();
+ let path = fixture.write_session("marker.jsonl", "marker");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let raw =
+ std::fs::read_to_string(agent_trace_outbox_entry_path(&fixture.agent_dir, &path)).unwrap();
+ assert_eq!(
+ parse_outbox_entry(&raw),
+ Some((path.to_string_lossy().into_owned(), None))
+ );
+ let signature = TraceUploadSignature::of(&path).unwrap();
+ record_agent_trace_outbox_upload(&fixture.agent_dir, &path, signature).unwrap();
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ Some(signature)
+ );
+}
+
+#[test]
+fn competing_delivery_is_nonblocking_and_crash_recoverable() {
+ let fixture = Fixture::new();
+ let path = fixture.write_session("lease.jsonl", "lease");
+ let lease = delivery_lease(&fixture.agent_dir, &path).unwrap();
+ assert!(delivery_lease(&fixture.agent_dir, &path).is_none());
+ drop(lease);
+ assert!(delivery_lease(&fixture.agent_dir, &path).is_some());
+}
+
+#[tokio::test(start_paused = true)]
+async fn controller_drop_cancels_timer_without_waiting_for_delivery() {
+ let fixture = Fixture::new();
+ let path = fixture.write_session("drop.jsonl", "drop");
+ let c = controller(&fixture, &path, true);
+ c.persisted(&path);
+ let cancel = c.cancel.clone();
+ let http = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));
+ let task = tokio::spawn(run_controller(
+ Arc::downgrade(&c),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ http,
+ None,
+ ));
+ drop(c);
+ assert!(cancel.is_cancelled());
+ task.await.unwrap();
+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).is_file());
+}
+
+// Informational default CI measurement; calibrated host benchmarks own latency gates.
+#[test]
+fn measure_synchronous_pending_marker_cost() {
+ let fixture = Fixture::new();
+ let mut cold = Vec::new();
+ let mut warm = Vec::new();
+ for index in 0..1000 {
+ let path = fixture.session_dir.join(format!("synthetic-{index}.jsonl"));
+ let c = controller(&fixture, &path, true);
+ let start = std::time::Instant::now();
+ c.persisted(&path);
+ cold.push(start.elapsed().as_nanos());
+ let start = std::time::Instant::now();
+ c.persisted(&path);
+ warm.push(start.elapsed().as_nanos());
+ }
+ cold.sort_unstable();
+ warm.sort_unstable();
+ let report = format!(
+ "{}\n",
+ json!({"samples":1000,"cold_marker_ns":{"p50":cold[500],"p95":cold[950],"p99":cold[990]},"warm_marker_ns":{"p50":warm[500],"p95":warm[950],"p99":warm[990]},"fsync":false,"consent_metadata_checks_included":false,"cached_consent_check_included":true})
+ );
+ // Explicit stdout retains this informational measurement in hosted CI logs
+ // even when the test harness captures println output for passing tests.
+ std::io::stdout().write_all(report.as_bytes()).unwrap();
+}
+
+fn enable_synthetic_fixture(fixture: &Fixture) {
+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);
+ settings.set_agent_traces_enabled(true).unwrap();
+ std::fs::write(
+ fixture.agent_dir.join("auth.json"),
+ r#"{"prime-agent-traces":{"type":"api_key","key":"synthetic-only"}}"#,
+ )
+ .unwrap();
+}
+
+struct ObservedSink {
+ requests: tokio::sync::mpsc::UnboundedSender<(
+ String,
+ tokio::sync::oneshot::Sender<TraceHttpResponse>,
+ )>,
+ cancelled: Arc<tokio::sync::Notify>,
+}
+
+impl TraceHttp for ObservedSink {
+ fn put<'a>(
+ &'a self,
+ _url: &'a str,
+ _headers: Vec<(String, String)>,
+ body: String,
+ _timeout_ms: u64,
+ cancel: Option<&'a TraceUploadCancel>,
+ ) -> std::pin::Pin<
+ Box<
+ dyn std::future::Future<Output = Result<TraceHttpResponse, TraceHttpError>> + Send + 'a,
+ >,
+ > {
+ Box::pin(async move {
+ let (reply, result) = tokio::sync::oneshot::channel();
+ self.requests.send((body, reply)).unwrap();
+ tokio::select! {
+ answer = result => Ok(answer.unwrap()),
+ () = async { if let Some(cancel) = cancel { cancel.wait().await; } else { std::future::pending::<()>().await; } } => {
+ self.cancelled.notify_one();
+ Err(TraceHttpError::Cancelled)
+ }
+ }
+ })
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn dropping_an_active_controller_cancels_a_hanging_sink() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("hanging.jsonl", "hanging");
+ let c = controller(&fixture, &path, true);
+ c.persisted(&path);
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let cancelled = Arc::new(tokio::sync::Notify::new());
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: cancelled.clone(),
+ });
+ let task = tokio::spawn(run_controller(
+ Arc::downgrade(&c),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ ));
+ let (_body, _reply) = observed.recv().await.unwrap();
+ drop(c);
+ cancelled.notified().await;
+ task.await.unwrap();
+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ None
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn disabling_consent_cancels_an_inflight_request_without_shutting_down_the_host() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("revoked-inflight.jsonl", "revoked");
+ let c = controller(&fixture, &path, true);
+ c.persisted(&path);
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let cancelled = Arc::new(tokio::sync::Notify::new());
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: cancelled.clone(),
+ });
+ let task = tokio::spawn(run_controller(
+ Arc::downgrade(&c),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ ));
+ let (_body, _reply) = observed.recv().await.unwrap();
+ let first_start = Instant::now();
+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);
+ settings.set_agent_traces_enabled(false).unwrap();
+ cancelled.notified().await;
+ assert!(!c.cancel.is_cancelled());
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ None
+ );
+ tokio::time::advance(MIN_INTERVAL * 2).await;
+ tokio::task::yield_now().await;
+ assert!(observed.try_recv().is_err(), "consent off must not deliver");
+ settings.set_agent_traces_enabled(true).unwrap();
+ let (_body, _reply) = tokio::time::timeout(MIN_INTERVAL * 2, observed.recv())
+ .await
+ .expect("restoring consent resumes without another persist")
+ .unwrap();
+ assert!(Instant::now() - first_start >= MIN_INTERVAL);
+ drop(c);
+ task.await.unwrap();
+}
+
+#[tokio::test(start_paused = true)]
+async fn recovery_reschedules_rate_limits_without_holding_an_upload_permit() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("recovery-retry.jsonl", "retry");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: Arc::new(tokio::sync::Notify::new()),
+ });
+ let permits = Arc::new(tokio::sync::Semaphore::new(1));
+ let task = tokio::spawn(recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ permits.clone(),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ ));
+ let (_body, first) = observed.recv().await.unwrap();
+ let started = Instant::now();
+ let mut limited = response(429, "{}");
+ limited.retry_after = Some("120".into());
+ first.send(limited).unwrap();
+ // Acquire readiness demonstrates the retry timer does not consume delivery capacity.
+ let permit = permits.acquire().await.unwrap();
+ drop(permit);
+ let (_body, second) = observed.recv().await.unwrap();
+ assert!(Instant::now() - started >= Duration::from_secs(120));
+ second.send(response(200, "{}")).unwrap();
+ assert!(task.await.unwrap());
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ TraceUploadSignature::of(&path)
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn recovery_shutdown_cancels_a_hanging_request_and_preserves_intent() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("recovery-shutdown.jsonl", "shutdown");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let cancelled = Arc::new(tokio::sync::Notify::new());
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: cancelled.clone(),
+ });
+ let cancel = TraceUploadCancel::new();
+ let task = tokio::spawn(recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ cancel.clone(),
+ ));
+ let (_body, _reply) = observed.recv().await.unwrap();
+ cancel.cancel();
+ cancelled.notified().await;
+ assert!(!task.await.unwrap());
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ None
+ );
+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());
+}
+
+#[tokio::test(start_paused = true)]
+async fn writes_during_upload_have_a_followup_and_do_not_advance_the_old_cursor() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("coalesce.jsonl", "coalesce");
+ let initial_signature = TraceUploadSignature::of(&path).unwrap();
+ let c = controller(&fixture, &path, true);
+ c.persisted(&path);
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: Arc::new(tokio::sync::Notify::new()),
+ });
+ let task = tokio::spawn(run_controller(
+ Arc::downgrade(&c),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ ));
+ let (first_body, first_reply) = observed.recv().await.unwrap();
+ let first_start = Instant::now();
+ {
+ use std::io::Write;
+ let mut file = std::fs::OpenOptions::new()
+ .append(true)
+ .open(&path)
+ .unwrap();
+ writeln!(
+ file,
+ "{}",
+ json!({"type":"custom","id":"synthetic-appended"})
+ )
+ .unwrap();
+ }
+ c.persisted(&path);
+ first_reply.send(response(200, "{}")).unwrap();
+ let (second_body, _second_reply) = observed.recv().await.unwrap();
+ assert!(Instant::now() - first_start >= MIN_INTERVAL);
+ assert!(!first_body.contains("synthetic-appended"));
+ assert!(second_body.contains("synthetic-appended"));
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ Some(initial_signature)
+ );
+ drop(c);
+ task.await.unwrap();
+}
+
+#[tokio::test(start_paused = true)]
+async fn opted_out_recovery_does_not_create_an_outbox() {
+ let fixture = Fixture::new();
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![]));
+ recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await;
+ assert!(!agent_trace_outbox_dir(&fixture.agent_dir).exists());
+}
+
+#[tokio::test(start_paused = true)]
+async fn startup_recovery_uploads_pending_and_prunes_missing_but_preserves_unknown_kinds() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("recovery.jsonl", "recovery");
+ let missing = fixture.session_dir.join("missing.jsonl");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ mark_pending(&fixture.agent_dir, &missing).unwrap();
+ let unknown = agent_trace_outbox_dir(&fixture.agent_dir).join("unknown.json");
+ std::fs::write(
+ &unknown,
+ json!({"sessionFile":path,"kind":"semantic-edges"}).to_string(),
+ )
+ .unwrap();
+ let malformed = agent_trace_outbox_dir(&fixture.agent_dir).join("malformed.json");
+ std::fs::write(&malformed, "{").unwrap();
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));
+ recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await;
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ TraceUploadSignature::of(&path)
+ );
+ assert!(!agent_trace_outbox_entry_path(&fixture.agent_dir, &missing).exists());
+ assert!(!malformed.exists());
+ assert!(unknown.exists());
+}
+
+#[test]
+fn persist_racing_a_pruner_retains_a_complete_nonblocking_fallback_marker() {
+ let fixture = Fixture::new();
+ let path = fixture.write_session("prune-race.jsonl", "race");
+ let entry = agent_trace_outbox_entry_path(&fixture.agent_dir, &path);
+ std::fs::create_dir_all(entry.parent().unwrap()).unwrap();
+ std::fs::write(&entry, "{").unwrap();
+ let pruner = outbox_mutation_lock(&entry).unwrap();
+ pruner.lock().unwrap();
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let fallback = entry.with_extension("pending.json");
+ let raw = std::fs::read_to_string(&fallback).unwrap();
+ assert_eq!(
+ parse_outbox_entry(&raw),
+ Some((path.to_string_lossy().into_owned(), None))
+ );
+ drop(pruner);
+ prune_entry(&entry, "{", None);
+ assert!(!entry.exists());
+ assert!(fallback.exists());
+ prune_entry(&fallback, &raw, None);
+ assert!(fallback.exists());
+}
+
+#[test]
+fn stale_prune_observations_never_remove_a_new_cursor_or_recreated_transcript() {
+ let fixture = Fixture::new();
+ let path = fixture.write_session("stale-prune.jsonl", "stale");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let entry = agent_trace_outbox_entry_path(&fixture.agent_dir, &path);
+ let raw = std::fs::read_to_string(&entry).unwrap();
+ let signature = TraceUploadSignature::of(&path).unwrap();
+ record_agent_trace_outbox_upload(&fixture.agent_dir, &path, signature).unwrap();
+ prune_entry(&entry, &raw, Some(&path));
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ Some(signature)
+ );
+ let raw = std::fs::read_to_string(&entry).unwrap();
+ prune_entry(&entry, &raw, Some(&path));
+ assert!(entry.exists());
+}
+
+#[tokio::test(start_paused = true)]
+async fn fallback_only_restart_schedules_the_live_controller_without_a_fresh_write() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("fallback-only.jsonl", "fallback-only");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let primary = agent_trace_outbox_entry_path(&fixture.agent_dir, &path);
+ let fallback = primary.with_extension("pending.json");
+ std::fs::rename(&primary, &fallback).unwrap();
+ let c = controller(&fixture, &path, true);
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: Arc::new(tokio::sync::Notify::new()),
+ });
+ let task = tokio::spawn(run_controller(
+ Arc::downgrade(&c),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ ));
+ let (body, _reply) = observed.recv().await.unwrap();
+ assert!(body.contains("fallback-only"));
+ drop(c);
+ task.await.unwrap();
+ assert!(fallback.exists());
+}
+
+#[test]
+fn recovery_registry_releases_inactive_directory_capacity_and_preserves_other_hosts() {
+ let fixture = Fixture::new();
+ let c = controller(&fixture, &fixture.session_dir.join("registry.jsonl"), true);
+ let another_host = controller(&fixture, &fixture.session_dir.join("other.jsonl"), true);
+ let dead_cancel = TraceUploadCancel::new();
+ let live_cancel = TraceUploadCancel::new();
+ let mut recovered = RecoveryRuns::new();
+ for n in 0..MAX_CONTROLLERS - 1 {
+ recovered.insert(
+ PathBuf::from(format!("synthetic-inactive-{n}")),
+ RecoveryRun {
+ cancel: dead_cancel.clone(),
+ ..RecoveryRun::new(vec![])
+ },
+ );
+ }
+ recovered.insert(
+ fixture.agent_dir,
+ RecoveryRun {
+ cancel: live_cancel.clone(),
+ ..RecoveryRun::new(vec![Arc::downgrade(&c), Arc::downgrade(&another_host)])
+ },
+ );
+ let registration = Registration::of(&c);
+ drop(c);
+ let _ = retain_live_recoveries(
+ &mut recovered,
+ &[registration, Registration::of(&another_host)],
+ );
+ assert!(dead_cancel.is_cancelled());
+ assert!(!live_cancel.is_cancelled());
+ assert_eq!(recovered.len(), 1);
+ drop(another_host);
+ let _ = retain_live_recoveries(&mut recovered, &[]);
+ assert!(live_cancel.is_cancelled());
+ assert!(recovered.is_empty());
+}
+
+#[tokio::test(start_paused = true)]
+async fn revoked_recovery_retains_intent_and_resumes_after_consent_restoration() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("revoked-recovery.jsonl", "revoked-recovery");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let cancelled = Arc::new(tokio::sync::Notify::new());
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: cancelled.clone(),
+ });
+ let permits = Arc::new(tokio::sync::Semaphore::new(1));
+ let task = tokio::spawn(recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ permits.clone(),
+ sink.clone(),
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ ));
+ let (_body, _reply) = observed.recv().await.unwrap();
+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);
+ settings.set_agent_traces_enabled(false).unwrap();
+ cancelled.notified().await;
+ assert!(!tokio::time::timeout(DEBOUNCE * 2, task)
+ .await
+ .unwrap()
+ .unwrap());
+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());
+ assert!(delivery_lease(&fixture.agent_dir, &path).is_some());
+ assert_eq!(permits.available_permits(), 1);
+ assert!(
+ !recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ permits.clone(),
+ sink.clone(),
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await
+ );
+ assert!(observed.try_recv().is_err());
+ settings.set_agent_traces_enabled(true).unwrap();
+ let task = tokio::spawn(recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ permits,
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ ));
+ let (_body, reply) = tokio::time::timeout(MIN_INTERVAL, observed.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ reply.send(response(200, "{}")).unwrap();
+ assert!(task.await.unwrap());
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ TraceUploadSignature::of(&path)
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn incomplete_recovery_cooldown_is_bounded_and_shutdown_cancellable() {
+ let state = Arc::new(std::sync::atomic::AtomicU8::new(1));
+ let cancel = TraceUploadCancel::new();
+ let task_state = state.clone();
+ let task_cancel = cancel.clone();
+ let task = tokio::spawn(async move {
+ settle_recovery_run(&task_state, false, None, &task_cancel).await;
+ });
+ tokio::task::yield_now().await;
+ tokio::time::advance(MIN_INTERVAL.checked_sub(DEBOUNCE).unwrap()).await;
+ assert_eq!(state.load(Ordering::Acquire), 1);
+ tokio::time::advance(DEBOUNCE).await;
+ task.await.unwrap();
+ assert_eq!(state.load(Ordering::Acquire), 0);
+ state.store(1, Ordering::Release);
+ let task_state = state.clone();
+ let task_cancel = cancel.clone();
+ let task = tokio::spawn(async move {
+ settle_recovery_run(&task_state, false, None, &task_cancel).await;
+ });
+ tokio::task::yield_now().await;
+ let began = Instant::now();
+ cancel.cancel();
+ task.await.unwrap();
+ assert_eq!(Instant::now(), began);
+ assert_eq!(state.load(Ordering::Acquire), 0);
+}
+
+fn mixed_consent_fixture() -> (Fixture, PathBuf, PathBuf, PathBuf) {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let first = fixture.write_session("disabled-project.jsonl", "disabled-project");
+ let second = fixture.write_session("enabled-project.jsonl", "enabled-project");
+ let second_cwd = fixture.agent_dir.join("other-project");
+ std::fs::create_dir_all(&second_cwd).unwrap();
+ for (path, cwd) in [(&first, &fixture.cwd), (&second, &second_cwd)] {
+ let raw = std::fs::read_to_string(path).unwrap();
+ let (header, tail) = raw.split_once('\n').unwrap();
+ let mut header: Value = serde_json::from_str(header).unwrap();
+ header["cwd"] = json!(cwd);
+ std::fs::write(path, format!("{header}\n{tail}")).unwrap();
+ mark_pending(&fixture.agent_dir, path).unwrap();
+ }
+ let config = fixture.cwd.join(crate::settings::storage::CONFIG_DIR_NAME);
+ std::fs::create_dir_all(&config).unwrap();
+ std::fs::write(
+ config.join("settings.json"),
+ r#"{"agentTraces":{"enabled":false}}"#,
+ )
+ .unwrap();
+ (fixture, first, second, second_cwd)
+}
+
+#[tokio::test(start_paused = true)]
+async fn disabled_project_recovery_does_not_block_other_opted_in_projects() {
+ let (fixture, first, second, second_cwd) = mixed_consent_fixture();
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));
+ assert!(
+ !recover(
+ second_cwd,
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await
+ );
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &first),
+ None
+ );
+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &first).exists());
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &second),
+ TraceUploadSignature::of(&second)
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn disabled_uploaded_cursor_does_not_rearm_recovery() {
+ let (fixture, first, second, second_cwd) = mixed_consent_fixture();
+ record_agent_trace_outbox_upload(
+ &fixture.agent_dir,
+ &first,
+ TraceUploadSignature::of(&first).unwrap(),
+ )
+ .unwrap();
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));
+ assert!(
+ recover(
+ second_cwd,
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await
+ );
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &second),
+ TraceUploadSignature::of(&second)
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn mixed_consent_rescan_preserves_exhausted_rate_limit_deadline() {
+ let (fixture, _first, _second, second_cwd) = mixed_consent_fixture();
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: Arc::new(tokio::sync::Notify::new()),
+ });
+ let task = tokio::spawn(recover_with_backoff(
+ second_cwd,
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ None,
+ ));
+ for _ in 0..3 {
+ let (_body, reply) = observed.recv().await.unwrap();
+ let mut limited = response(429, "{}");
+ limited.retry_after = Some("120".into());
+ reply.send(limited).unwrap();
+ }
+ let (complete, not_before) = task.await.unwrap();
+ assert!(!complete);
+ let began = Instant::now();
+ assert_eq!(not_before, Some(began + Duration::from_secs(120)));
+ let state = Arc::new(std::sync::atomic::AtomicU8::new(1));
+ let task_state = state.clone();
+ let task = tokio::spawn(async move {
+ settle_recovery_run(&task_state, complete, not_before, &TraceUploadCancel::new()).await;
+ });
+ tokio::task::yield_now().await;
+ tokio::time::advance(MIN_INTERVAL).await;
+ assert_eq!(
+ state.load(Ordering::Acquire),
+ 1,
+ "must not rescan after only 60 seconds"
+ );
+ tokio::time::advance(MIN_INTERVAL).await;
+ task.await.unwrap();
+ assert_eq!(state.load(Ordering::Acquire), 0);
+ assert_eq!(Instant::now(), began + Duration::from_secs(120));
+}
+
+#[test]
+fn oversized_cursor_records_fail_closed_with_bounded_reads() {
+ let fixture = Fixture::new();
+ let path = fixture.write_session("large-cursor.jsonl", "large-cursor");
+ record_agent_trace_outbox_upload(
+ &fixture.agent_dir,
+ &path,
+ TraceUploadSignature::of(&path).unwrap(),
+ )
+ .unwrap();
+ let entry = agent_trace_outbox_entry_path(&fixture.agent_dir, &path);
+ let mut value: Value = serde_json::from_str(&std::fs::read_to_string(&entry).unwrap()).unwrap();
+ value["padding"] = json!(" ".repeat(64 * 1024));
+ std::fs::write(entry, value.to_string()).unwrap();
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ None
+ );
+}
+
+#[tokio::test(start_paused = true)]
+// Credential lookup must stay stable while synthetic background workers run.
+#[allow(clippy::await_holding_lock)]
+async fn replacement_at_capacity_reuses_a_slot_and_preserves_late_retirement() {
+ let _env = env_lock();
+ let _credentials = clear_trace_credentials();
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let original_path = fixture.write_session("original.jsonl", "original");
+ let replacement_path = fixture.write_session("replacement.jsonl", "replacement");
+ let source = controller(&fixture, &original_path, true);
+ source.persisted(&original_path);
+ let service = Service {
+ controllers: Mutex::new(Vec::new()),
+ wake: tokio::sync::Notify::new(),
+ };
+ assert!(service.register(&source));
+ let mut hosts = vec![source.clone()];
+ for n in 1..MAX_CONTROLLERS {
+ let host = controller(
+ &fixture,
+ &fixture.session_dir.join(format!("host-{n}.jsonl")),
+ false,
+ );
+ assert!(service.register(&host));
+ hosts.push(host);
+ }
+ let old_sink = Arc::new(ScriptedTraceHttp::new(Vec::new()));
+ let old_task = tokio::spawn(run_controller(
+ Arc::downgrade(&source),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ old_sink.clone(),
+ Some("http://synthetic.invalid".into()),
+ ));
+ let first = controller(&fixture, &replacement_path, true);
+ assert!(service.replace(&source, &first));
+ assert!(source.cancel.is_cancelled());
+ old_task.await.unwrap();
+ assert_eq!(old_sink.request_count(), 0);
+ first.persisted(&replacement_path);
+ let mut recovered = RecoveryRuns::new();
+ let run = RecoveryRun::new(hosts.iter().map(Arc::downgrade).collect());
+ run.state.store(2, Ordering::Release);
+ recovered.insert(fixture.agent_dir.clone(), run);
+ let snapshot = service.controllers.lock().unwrap().clone();
+ retain_live_recoveries(&mut recovered, &snapshot);
+ assert!(recovered[&fixture.agent_dir]
+ .replay_needed
+ .load(Ordering::Acquire));
+ let latest = controller(&fixture, &replacement_path, true);
+ assert!(service.replace(&first, &latest));
+ assert!(first.cancel.is_cancelled());
+ acknowledge_replacements(&mut service.controllers.lock().unwrap(), &snapshot);
+ assert!(service
+ .controllers
+ .lock()
+ .unwrap()
+ .iter()
+ .any(|registration| {
+ registration.controller.ptr_eq(&Arc::downgrade(&latest))
+ && registration.retired_predecessor
+ }));
+ let current = service.controllers.lock().unwrap().clone();
+ retain_live_recoveries(&mut recovered, &current);
+ acknowledge_replacements(&mut service.controllers.lock().unwrap(), &current);
+ assert_eq!(service.controllers.lock().unwrap().len(), MAX_CONTROLLERS);
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));
+ let (complete, _) = recover_with_backoff(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink.clone(),
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ Some(vec![Arc::downgrade(&latest)]),
+ )
+ .await;
+ assert!(!complete); // Replacement owns its marker; the predecessor was recovered.
+ assert_eq!(sink.request_count(), 1);
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &original_path),
+ TraceUploadSignature::of(&original_path)
+ );
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let task = tokio::spawn(run_controller(
+ Arc::downgrade(&latest),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ Arc::new(ObservedSink {
+ requests,
+ cancelled: Arc::new(tokio::sync::Notify::new()),
+ }),
+ Some("http://synthetic.invalid".into()),
+ ));
+ let (_body, reply) = tokio::time::timeout(MIN_INTERVAL, observed.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ reply.send(response(200, "{}")).unwrap();
+ drop(latest);
+ task.await.unwrap();
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_busy_delivery_lease_leaves_recovery_incomplete_until_release() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("leased-recovery.jsonl", "leased-recovery");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let lease = delivery_lease(&fixture.agent_dir, &path).unwrap();
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));
+ assert!(
+ !recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink.clone(),
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await,
+ "a leased marker must keep the sweep incomplete so recovery re-arms"
+ );
+ drop(lease);
+ assert!(
+ recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await
+ );
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ TraceUploadSignature::of(&path)
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn exhausted_recovery_retries_leave_the_sweep_incomplete() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("exhausted-recovery.jsonl", "exhausted-recovery");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ // Three bounded sweep attempts each exhaust the transport retries.
+ let sink = Arc::new(ScriptedTraceHttp::new(
+ std::iter::repeat_n(Ok(response(503, "{}")), 12).collect(),
+ ));
+ assert!(
+ !recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await,
+ "work still due after the bounded retries must keep the sweep incomplete"
+ );
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ None
+ );
+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());
+}
+
+#[tokio::test(start_paused = true)]
+// The env lock is held across awaits by design (env must stay stable).
+#[allow(clippy::await_holding_lock)]
+async fn a_credentialless_controller_retries_after_credentials_arrive() {
+ let _env = env_lock();
+ let _credentials = clear_trace_credentials();
+ let fixture = Fixture::new();
+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);
+ settings.set_agent_traces_enabled(true).unwrap();
+ // Deliberately no auth.json yet: the delivery fails closed, and the live
+ // controller must schedule its own retry rather than park forever.
+ let path = fixture.write_session("late-login.jsonl", "late-login");
+ let c = controller(&fixture, &path, true);
+ c.persisted(&path);
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: Arc::new(tokio::sync::Notify::new()),
+ });
+ let task = tokio::spawn(run_controller(
+ Arc::downgrade(&c),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ ));
+ // Park so the controller really attempts at +1s against the missing
+ // credentials before any write makes one deliverable.
+ tokio::time::sleep(MIN_INTERVAL + DEBOUNCE * 2).await;
+ std::fs::write(
+ fixture.agent_dir.join("auth.json"),
+ r#"{"prime-agent-traces":{"type":"api_key","key":"synthetic-only"}}"#,
+ )
+ .unwrap();
+ let (_body, reply) = tokio::time::timeout(MIN_INTERVAL * 2, observed.recv())
+ .await
+ .expect("a credentialless delivery must retry after credentials arrive")
+ .unwrap();
+ reply.send(response(200, "{}")).unwrap();
+ drop(c);
+ task.await.unwrap();
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_settings_change_after_load_still_records_the_write_intent() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("settings-race.jsonl", "settings-race");
+ let c = controller(&fixture, &path, true);
+ // A concurrent project edit changes the settings metadata after the
+ // host captured its consent snapshot: removal is unambiguous.
+ std::fs::remove_file(fixture.agent_dir.join("settings.json")).unwrap();
+ c.persisted(&path);
+ assert!(
+ agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists(),
+ "the write intent must stay durable across a concurrent settings edit"
+ );
+ // The worker re-reads consent before delivery; restore the opted-in edit.
+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);
+ settings.set_agent_traces_enabled(true).unwrap();
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: Arc::new(tokio::sync::Notify::new()),
+ });
+ let task = tokio::spawn(run_controller(
+ Arc::downgrade(&c),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ ));
+ let (_body, reply) = tokio::time::timeout(MIN_INTERVAL, observed.recv())
+ .await
+ .expect("the worker revalidates consent and delivers the retained intent")
+ .unwrap();
+ reply.send(response(200, "{}")).unwrap();
+ drop(c);
+ task.await.unwrap();
+}
+
+#[tokio::test(start_paused = true)]
+// The env lock is held across awaits by design (env must stay stable).
+#[allow(clippy::await_holding_lock)]
+async fn missing_credentials_leave_the_sweep_incomplete_until_provided() {
+ let _env = env_lock();
+ let _credentials = clear_trace_credentials();
+ let fixture = Fixture::new();
+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);
+ settings.set_agent_traces_enabled(true).unwrap();
+ // Deliberately no auth.json yet: the upload must fail closed.
+ let path = fixture.write_session("no-credentials.jsonl", "no-credentials");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));
+ assert!(
+ !recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink.clone(),
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await,
+ "missing credentials must keep the sweep incomplete so it re-arms"
+ );
+ std::fs::write(
+ fixture.agent_dir.join("auth.json"),
+ r#"{"prime-agent-traces":{"type":"api_key","key":"synthetic-only"}}"#,
+ )
+ .unwrap();
+ assert!(
+ recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await
+ );
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ TraceUploadSignature::of(&path)
+ );
+}
+
+#[tokio::test(start_paused = true)]
+// Keep credential lookup stable across injected background I/O failures.
+#[allow(clippy::await_holding_lock)]
+async fn transient_recovery_reads_rearm_and_deliver_after_restoration() {
+ let _env = env_lock();
+ let _credentials = clear_trace_credentials();
+ for stage in [
+ RecoveryRead::Directory,
+ RecoveryRead::Iteration,
+ RecoveryRead::RecordMetadata,
+ RecoveryRead::Record,
+ RecoveryRead::SessionMetadata,
+ RecoveryRead::Header,
+ ] {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("transient.jsonl", "transient");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));
+ let (complete, _) = recover_with_io(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink.clone(),
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ Some(Vec::new()),
+ |read, _| {
+ if read == stage {
+ Err(std::io::Error::from(std::io::ErrorKind::Interrupted))
+ } else {
+ Ok(())
+ }
+ },
+ )
+ .await;
+ assert!(!complete, "{stage:?} must keep recovery incomplete");
+ assert!(agent_trace_outbox_entry_path(&fixture.agent_dir, &path).exists());
+ assert_eq!(sink.request_count(), 0);
+ assert!(
+ recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink.clone(),
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await
+ );
+ assert_eq!(sink.request_count(), 1);
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ TraceUploadSignature::of(&path)
+ );
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn terminal_invalid_records_and_headers_complete_without_network_or_rescan() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let valid = fixture.write_session("valid.jsonl", "valid");
+ let header = std::fs::read_to_string(&valid).unwrap();
+ for (name, bytes) in [
+ ("malformed", b"not a header\n".to_vec()),
+ ("utf8", vec![0xff, b'\n']),
+ ("empty", Vec::new()),
+ (
+ "oversized",
+ [
+ header.lines().next().unwrap().as_bytes(),
+ &vec![b' '; 256 * 1024],
+ ]
+ .concat(),
+ ),
+ ] {
+ let path = fixture.session_dir.join(format!("{name}.jsonl"));
+ std::fs::write(&path, bytes).unwrap();
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ }
+ let outbox = agent_trace_outbox_dir(&fixture.agent_dir);
+ std::fs::write(outbox.join("invalid-utf8.json"), [0xff]).unwrap();
+ std::fs::write(outbox.join("oversized.json"), vec![b' '; 64 * 1024 + 1]).unwrap();
+ std::fs::write(outbox.join("malformed.json"), "{").unwrap();
+ let sink = Arc::new(ScriptedTraceHttp::new(Vec::new()));
+ let (complete, not_before) = recover_with_backoff(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink.clone(),
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ Some(Vec::new()),
+ )
+ .await;
+ assert_eq!((complete, not_before), (true, None));
+ let state = std::sync::atomic::AtomicU8::new(1);
+ settle_recovery_run(&state, complete, not_before, &TraceUploadCancel::new()).await;
+ assert_eq!(state.load(Ordering::Acquire), 2);
+ assert_eq!(sink.request_count(), 0);
+}
+
+#[tokio::test(start_paused = true)]
+// The env lock is held across awaits to keep credential lookup deterministic.
+#[allow(clippy::await_holding_lock)]
+async fn bounded_admission_preserves_retirement_and_delivers_on_readmission() {
+ let _env = env_lock();
+ let _credentials = clear_trace_credentials();
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("admission.jsonl", "admission");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let service = Service {
+ controllers: Mutex::new(Vec::new()),
+ wake: tokio::sync::Notify::new(),
+ };
+ let mut hosts = Vec::new();
+ for n in 0..MAX_CONTROLLERS {
+ let host = controller(
+ &fixture,
+ &fixture.session_dir.join(format!("host-{n}.jsonl")),
+ false,
+ );
+ assert!(service.register(&host));
+ hosts.push(host);
+ }
+ let candidate = controller(&fixture, &path, true);
+ assert!(!service.register(&candidate));
+ assert_eq!(service.controllers.lock().unwrap().len(), MAX_CONTROLLERS);
+ let mut recovered = RecoveryRuns::new();
+ let run = RecoveryRun::new(hosts.iter().map(Arc::downgrade).collect());
+ run.state.store(2, Ordering::Release);
+ recovered.insert(fixture.agent_dir.clone(), run);
+ drop(hosts.pop().unwrap());
+ // Retirement cannot be discarded by foreground admission before replay sees it.
+ assert!(!service.register(&candidate));
+ let snapshot = service.controllers.lock().unwrap().clone();
+ let retired = retain_live_recoveries(&mut recovered, &snapshot);
+ assert!(recovered[&fixture.agent_dir]
+ .replay_needed
+ .load(Ordering::Acquire));
+ assert_eq!(retired.len(), 1);
+ acknowledge_retired_registrations(&mut service.controllers.lock().unwrap(), &retired);
+ assert!(service.register(&candidate));
+ assert_eq!(service.controllers.lock().unwrap().len(), MAX_CONTROLLERS);
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: Arc::new(tokio::sync::Notify::new()),
+ });
+ let task = tokio::spawn(run_controller(
+ Arc::downgrade(&candidate),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ ));
+ // Admission resumes the durable marker with no restart or fresh persist.
+ let (_body, reply) = tokio::time::timeout(MIN_INTERVAL, observed.recv())
+ .await
+ .expect("readmitted controller must deliver existing intent")
+ .unwrap();
+ reply.send(response(200, "{}")).unwrap();
+ drop(candidate);
+ task.await.unwrap();
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_busy_delivery_lease_does_not_hold_upload_capacity() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let leased = fixture.write_session("lease-wait.jsonl", "lease-wait");
+ let free = fixture.write_session("lease-free.jsonl", "lease-free");
+ let busy_host = controller(&fixture, &leased, true);
+ busy_host.persisted(&leased);
+ let lease = delivery_lease(&fixture.agent_dir, &leased).unwrap();
+ let (requests, mut observed) = tokio::sync::mpsc::unbounded_channel();
+ let sink = Arc::new(ObservedSink {
+ requests,
+ cancelled: Arc::new(tokio::sync::Notify::new()),
+ });
+ let permits = Arc::new(tokio::sync::Semaphore::new(1));
+ let busy_task = tokio::spawn(run_controller(
+ Arc::downgrade(&busy_host),
+ permits.clone(),
+ sink.clone(),
+ Some("http://synthetic.invalid".into()),
+ ));
+ // The busy host's due (+1s) must strictly precede the free host's (+1.5s):
+ // startup re-persists each deadline from its own spawn instant, so the
+ // ordering is built from distinct spawn instants, never wake order.
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ let free_host = controller(&fixture, &free, true);
+ free_host.persisted(&free);
+ let free_task = tokio::spawn(run_controller(
+ Arc::downgrade(&free_host),
+ permits.clone(),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ ));
+ // The busy host attempts the held lease at +1s and must release the shared
+ // permit while it waits, or the free host only delivers at its +2s retry.
+ let (body, reply) =
+ tokio::time::timeout(DEBOUNCE + Duration::from_millis(250), observed.recv())
+ .await
+ .expect("a busy lease must not starve an unrelated delivery")
+ .unwrap();
+ assert!(
+ body.contains("lease-free"),
+ "the free session delivers first"
+ );
+ reply.send(response(200, "{}")).unwrap();
+ drop(lease);
+ let (body, reply) = tokio::time::timeout(MIN_INTERVAL, observed.recv())
+ .await
+ .expect("the leased session retries after the lease releases")
+ .unwrap();
+ assert!(body.contains("lease-wait"));
+ reply.send(response(200, "{}")).unwrap();
+ drop(busy_host);
+ drop(free_host);
+ busy_task.await.unwrap();
+ free_task.await.unwrap();
+}
+
+#[tokio::test(start_paused = true)]
+async fn pending_live_owner_keeps_recovery_incomplete_until_retirement() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("live-retirement.jsonl", "live-retirement");
+ mark_pending(&fixture.agent_dir, &path).unwrap();
+ let c = controller(&fixture, &path, true);
+ let weak = Arc::downgrade(&c);
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![]));
+ let (complete, _) = recover_with_backoff(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ Some(vec![weak]),
+ )
+ .await;
+ assert!(!complete);
+ drop(c);
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));
+ assert!(
+ recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await
+ );
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ TraceUploadSignature::of(&path)
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn host_retirement_rearms_an_already_complete_group_for_later_markers() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let path = fixture.write_session("late-marker.jsonl", "late-marker");
+ let c = controller(&fixture, &path, true);
+ let another = controller(&fixture, &fixture.session_dir.join("survivor.jsonl"), true);
+ let weak = Arc::downgrade(&c);
+ let another_weak = Arc::downgrade(&another);
+ let mut runs = RecoveryRuns::new();
+ let run = RecoveryRun::new(vec![weak.clone(), another_weak.clone()]);
+ run.state.store(2, Ordering::Release);
+ runs.insert(fixture.agent_dir.clone(), run);
+ let registration = Registration::of(&c);
+ let another_registration = Registration::of(&another);
+ c.persisted(&path);
+ drop(c);
+ let _ = retain_live_recoveries(&mut runs, &[registration, another_registration]);
+ let run = runs.get(&fixture.agent_dir).unwrap();
+ assert!(run.replay_needed.load(Ordering::Acquire));
+ assert!(!run.cancel.is_cancelled());
+ rearm_retired_recovery(run);
+ assert_eq!(run.state.load(Ordering::Acquire), 1);
+ tokio::task::yield_now().await;
+ tokio::time::advance(MIN_INTERVAL).await;
+ tokio::task::yield_now().await;
+ assert_eq!(run.state.load(Ordering::Acquire), 0);
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![Ok(response(200, "{}"))]));
+ assert!(
+ recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await
+ );
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, &path),
+ TraceUploadSignature::of(&path)
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn retirement_racing_completion_survives_and_preserves_the_retry_deadline() {
+ let run = RecoveryRun::new(vec![]);
+ run.state.store(1, Ordering::Release);
+ let complete = !run.replay_needed.load(Ordering::Acquire);
+ run.replay_needed.store(true, Ordering::Release);
+ let deadline = Instant::now() + Duration::from_secs(120);
+ *run.not_before.lock().unwrap() = Some(deadline);
+ settle_recovery_run(&run.state, complete, Some(deadline), &run.cancel).await;
+ assert_eq!(run.state.load(Ordering::Acquire), 2);
+ rearm_retired_recovery(&run);
+ rearm_retired_recovery(&run);
+ assert_eq!(run.state.load(Ordering::Acquire), 1);
+ tokio::task::yield_now().await;
+ tokio::time::advance(MIN_INTERVAL).await;
+ assert_eq!(run.state.load(Ordering::Acquire), 1);
+ tokio::time::advance(MIN_INTERVAL).await;
+ tokio::task::yield_now().await;
+ assert_eq!(run.state.load(Ordering::Acquire), 0);
+ assert!(
+ run.replay_needed.load(Ordering::Acquire),
+ "only a new sweep may clear retirement intent"
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn between_tick_retirement_and_late_acknowledgement_preserve_pending_markers() {
+ let fixture = Fixture::new();
+ enable_synthetic_fixture(&fixture);
+ let survivor = controller(&fixture, &fixture.session_dir.join("survivor.jsonl"), true);
+ let run = RecoveryRun::new(vec![Arc::downgrade(&survivor)]);
+ run.state.store(2, Ordering::Release);
+ let mut runs = RecoveryRuns::new();
+ runs.insert(fixture.agent_dir.clone(), run);
+ let mut registrations = vec![Registration::of(&survivor)];
+ let first_path = fixture.write_session("between-ticks.jsonl", "between-ticks");
+ let first = controller(&fixture, &first_path, true);
+ registrations.push(Registration::of(&first));
+ first.persisted(&first_path);
+ drop(first);
+ let retired = retain_live_recoveries(&mut runs, &registrations);
+ assert_eq!(retired.len(), 1);
+ let second_path = fixture.write_session("after-snapshot.jsonl", "after-snapshot");
+ let second = controller(&fixture, &second_path, true);
+ registrations.push(Registration::of(&second));
+ second.persisted(&second_path);
+ drop(second);
+ acknowledge_retired_registrations(&mut registrations, &retired);
+ assert_eq!(
+ registrations.len(),
+ 2,
+ "a retirement after the snapshot must remain until the next tick"
+ );
+ let retired = retain_live_recoveries(&mut runs, &registrations);
+ assert_eq!(retired.len(), 1);
+ acknowledge_retired_registrations(&mut registrations, &retired);
+ assert_eq!(registrations.len(), 1);
+ let run = runs.get(&fixture.agent_dir).unwrap();
+ assert!(run.replay_needed.load(Ordering::Acquire));
+ rearm_retired_recovery(run);
+ tokio::task::yield_now().await;
+ tokio::time::advance(MIN_INTERVAL).await;
+ tokio::task::yield_now().await;
+ assert_eq!(run.state.load(Ordering::Acquire), 0);
+ let sink = Arc::new(ScriptedTraceHttp::new(vec![
+ Ok(response(200, "{}")),
+ Ok(response(200, "{}")),
+ ]));
+ assert!(
+ recover(
+ fixture.cwd.clone(),
+ fixture.agent_dir.clone(),
+ Arc::new(tokio::sync::Semaphore::new(1)),
+ sink,
+ Some("http://synthetic.invalid".into()),
+ TraceUploadCancel::new(),
+ )
+ .await
+ );
+ for path in [&first_path, &second_path] {
+ assert_eq!(
+ read_agent_trace_outbox_entry(&fixture.agent_dir, path),
+ TraceUploadSignature::of(path)
+ );
+ }
+}
diff --git a/crates/pa-core/src/agent_traces/tests.rs b/crates/pa-core/src/agent_traces/tests.rs
index 9504fce826..988fc34615 100644
--- a/crates/pa-core/src/agent_traces/tests.rs
+++ b/crates/pa-core/src/agent_traces/tests.rs
@@ -14,13 +14,13 @@ type CapturedRequest = (String, Vec<(String, String)>, String);
/// A scripted transport: one PUT slot at a time with its captured
/// request and scripted answer.
-struct ScriptedTraceHttp {
+pub(super) struct ScriptedTraceHttp {
requests: Mutex<Vec<CapturedRequest>>,
answers: Mutex<VecDeque<Result<TraceHttpResponse, TraceHttpError>>>,
}
impl ScriptedTraceHttp {
- fn new(answers: Vec<Result<TraceHttpResponse, TraceHttpError>>) -> Self {
+ pub(super) fn new(answers: Vec<Result<TraceHttpResponse, TraceHttpError>>) -> Self {
ScriptedTraceHttp {
requests: Mutex::new(Vec::new()),
answers: Mutex::new(answers.into_iter().collect()),
@@ -30,6 +30,10 @@ impl ScriptedTraceHttp {
fn last_request(&self) -> CapturedRequest {
self.requests.lock().unwrap().last().cloned().unwrap()
}
+
+ pub(super) fn request_count(&self) -> usize {
+ self.requests.lock().unwrap().len()
+ }
}
impl TraceHttp for ScriptedTraceHttp {
@@ -48,7 +52,7 @@ impl TraceHttp for ScriptedTraceHttp {
}
}
-fn response(status: u16, body: &str) -> TraceHttpResponse {
+pub(super) fn response(status: u16, body: &str) -> TraceHttpResponse {
TraceHttpResponse {
status,
body: body.to_string(),
@@ -56,17 +60,17 @@ fn response(status: u16, body: &str) -> TraceHttpResponse {
}
}
-struct Fixture {
+pub(super) struct Fixture {
/// The temp dir stays alive for the fixture's life (the paths
/// point into it); it is never read.
_dir: tempfile::TempDir,
- cwd: PathBuf,
- agent_dir: PathBuf,
- session_dir: PathBuf,
+ pub(super) cwd: PathBuf,
+ pub(super) agent_dir: PathBuf,
+ pub(super) session_dir: PathBuf,
}
impl Fixture {
- fn new() -> Self {
+ pub(super) fn new() -> Self {
let dir = tempfile::tempdir().expect("temp dir");
let agent_dir = dir.path().join("agent");
let session_dir = agent_dir.join("sessions");
@@ -79,7 +83,7 @@ impl Fixture {
}
}
- fn write_session(&self, name: &str, id: &str) -> PathBuf {
+ pub(super) fn write_session(&self, name: &str, id: &str) -> PathBuf {
let path = self.session_dir.join(name);
std::fs::write(
&path,
@@ -113,12 +117,42 @@ impl Fixture {
/// The engine reads process env (the credential keys); the tests that
/// touch it serialize on one lock.
-fn env_lock() -> std::sync::MutexGuard<'static, ()> {
+pub(super) fn env_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
LOCK.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
+/// Clears the credential env vars under the lock and restores the prior
+/// values (or absence) on drop — including on assertion panics.
+pub(super) struct TraceEnvCredentials {
+ saved: [(&'static str, Option<std::ffi::OsString>); 2],
+}
+
+pub(super) fn clear_trace_credentials() -> TraceEnvCredentials {
+ let saved = [
+ (
+ "PRIME_AGENT_TRACES_API_KEY",
+ std::env::var_os("PRIME_AGENT_TRACES_API_KEY"),
+ ),
+ ("PRIME_API_KEY", std::env::var_os("PRIME_API_KEY")),
+ ];
+ std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
+ std::env::remove_var("PRIME_API_KEY");
+ TraceEnvCredentials { saved }
+}
+
+impl Drop for TraceEnvCredentials {
+ fn drop(&mut self) {
+ for (name, value) in &self.saved {
+ match value {
+ Some(value) => std::env::set_var(name, value),
+ None => std::env::remove_var(name),
+ }
+ }
+ }
+}
+
#[test]
fn the_header_validation_matches_ts() {
let fixture = Fixture::new();
@@ -143,8 +177,7 @@ fn the_header_validation_matches_ts() {
#[allow(clippy::await_holding_lock)]
async fn the_upload_sends_the_ts_request_and_records_the_cursor() {
let _env = env_lock();
- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
- std::env::remove_var("PRIME_API_KEY");
+ let _credentials = clear_trace_credentials();
let fixture = Fixture::new();
let session = fixture.write_session("s.jsonl", "sid-1");
std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");
@@ -186,7 +219,6 @@ async fn the_upload_sends_the_ts_request_and_records_the_cursor() {
assert!(signature_equals(recorded, signature));
let log = std::fs::read_to_string(agent_traces_log_path(&fixture.agent_dir)).expect("log");
assert!(log.contains("uploaded session sid-1 (42 bytes)"), "{log}");
- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
}
#[tokio::test]
@@ -212,8 +244,7 @@ async fn the_disabled_requirement_gate_matches_ts() {
#[allow(clippy::await_holding_lock)]
async fn a_missing_credential_short_circuits_the_request() {
let _env = env_lock();
- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
- std::env::remove_var("PRIME_API_KEY");
+ let _credentials = clear_trace_credentials();
let fixture = Fixture::new();
let session = fixture.write_session("s.jsonl", "sid");
let http = ScriptedTraceHttp::new(vec![]);
@@ -226,6 +257,7 @@ async fn a_missing_credential_short_circuits_the_request() {
#[allow(clippy::await_holding_lock)]
async fn an_oversize_session_reports_the_limit() {
let _env = env_lock();
+ let _credentials = clear_trace_credentials();
std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");
let fixture = Fixture::new();
let session = fixture.session_dir.join("big.jsonl");
@@ -251,7 +283,6 @@ async fn an_oversize_session_reports_the_limit() {
max_bytes: MAX_TRACE_BYTES,
}
);
- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
}
#[tokio::test]
@@ -259,6 +290,7 @@ async fn an_oversize_session_reports_the_limit() {
#[allow(clippy::await_holding_lock)]
async fn an_error_response_carries_the_status_and_message() {
let _env = env_lock();
+ let _credentials = clear_trace_credentials();
std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");
let fixture = Fixture::new();
let session = fixture.write_session("s.jsonl", "sid");
@@ -272,7 +304,6 @@ async fn an_error_response_carries_the_status_and_message() {
retry_after_ms: None,
}
);
- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
}
#[tokio::test]
@@ -280,6 +311,7 @@ async fn an_error_response_carries_the_status_and_message() {
#[allow(clippy::await_holding_lock)]
async fn the_retriable_statuses_back_off_and_503_honors_retry_after() {
let _env = env_lock();
+ let _credentials = clear_trace_credentials();
std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");
let fixture = Fixture::new();
let session = fixture.write_session("s.jsonl", "sid");
@@ -306,7 +338,6 @@ async fn the_retriable_statuses_back_off_and_503_honors_retry_after() {
assert!(matches!(result, TraceUploadResult::Uploaded { .. }));
// The Retry-After second won over the exponential backoff.
assert_eq!(delays.load(Ordering::SeqCst), 1000);
- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
}
#[tokio::test]
@@ -314,6 +345,7 @@ async fn the_retriable_statuses_back_off_and_503_honors_retry_after() {
#[allow(clippy::await_holding_lock)]
async fn the_outbox_cursor_makes_an_enabled_upload_unchanged() {
let _env = env_lock();
+ let _credentials = clear_trace_credentials();
std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");
let fixture = Fixture::new();
let session = fixture.write_session("s.jsonl", "sid");
@@ -331,7 +363,6 @@ async fn the_outbox_cursor_makes_an_enabled_upload_unchanged() {
options.require_enabled = true;
let result = upload_trace_file(&options).await;
assert_eq!(result, TraceUploadResult::Unchanged);
- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
}
#[tokio::test]
@@ -485,6 +516,7 @@ fn format_http_date(ms: u64) -> String {
#[allow(clippy::await_holding_lock)]
async fn the_upload_all_sweeps_with_the_gate_and_tallies() {
let _env = env_lock();
+ let _credentials = clear_trace_credentials();
std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");
let fixture = Fixture::new();
fixture.write_session("a.jsonl", "sid-a");
@@ -525,7 +557,6 @@ async fn the_upload_all_sweeps_with_the_gate_and_tallies() {
vec![(0, 1), (1, 1)],
"the progress reports completion in file order"
);
- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
}
#[tokio::test(start_paused = true)]
@@ -533,6 +564,7 @@ async fn the_upload_all_sweeps_with_the_gate_and_tallies() {
#[allow(clippy::await_holding_lock)]
async fn the_rate_gate_reports_the_wait_and_serializes() {
let _env = env_lock();
+ let _credentials = clear_trace_credentials();
std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");
let fixture = Fixture::new();
let http = ScriptedTraceHttp::new(vec![]);
@@ -563,7 +595,6 @@ async fn the_rate_gate_reports_the_wait_and_serializes() {
.await
.expect("the second slot");
assert!(delays.load(Ordering::SeqCst) >= TRACE_UPLOAD_ALL_MIN_REQUEST_INTERVAL_MS - 1000);
- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
}
#[tokio::test]
@@ -571,6 +602,7 @@ async fn the_rate_gate_reports_the_wait_and_serializes() {
#[allow(clippy::await_holding_lock)]
async fn the_cancel_stops_the_sweep_between_files() {
let _env = env_lock();
+ let _credentials = clear_trace_credentials();
std::env::set_var("PRIME_AGENT_TRACES_API_KEY", "trace-key");
let fixture = Fixture::new();
fixture.write_session("a.jsonl", "sid-a");
@@ -596,7 +628,6 @@ async fn the_cancel_stops_the_sweep_between_files() {
assert_eq!(result.total, 2);
assert_eq!(result.uploaded, 0);
assert_eq!(result.skipped, 2);
- std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
}
#[test]
@@ -623,3 +654,29 @@ fn the_outbox_entry_path_is_the_path_hash() {
path
);
}
+
+#[tokio::test(start_paused = true)]
+async fn consent_revoked_during_backoff_prevents_the_next_request() {
+ let fixture = Fixture::new();
+ let mut settings = crate::settings::SettingsManager::create(&fixture.cwd, &fixture.agent_dir);
+ settings.set_agent_traces_enabled(true).unwrap();
+ let http = ScriptedTraceHttp::new(vec![Ok(response(500, "synthetic failure"))]);
+ let cwd = fixture.cwd.clone();
+ let agent_dir = fixture.agent_dir.clone();
+ let mut options = fixture.options(&http, None);
+ options.require_enabled = true;
+ options.on_upload_delay = Some(Arc::new(move |_| {
+ let mut settings = crate::settings::SettingsManager::create(&cwd, &agent_dir);
+ settings.set_agent_traces_enabled(false).unwrap();
+ }));
+ let result = super::upload::fetch_with_retry(
+ &options,
+ "http://synthetic.invalid",
+ vec![],
+ String::new(),
+ None,
+ )
+ .await;
+ assert_eq!(result, Err(TraceHttpError::Cancelled));
+ assert_eq!(http.requests.lock().unwrap().len(), 1);
+}
diff --git a/crates/pa-core/src/agent_traces/upload.rs b/crates/pa-core/src/agent_traces/upload.rs
index fd27a8cba4..ca657f1550 100644
--- a/crates/pa-core/src/agent_traces/upload.rs
+++ b/crates/pa-core/src/agent_traces/upload.rs
@@ -37,7 +37,7 @@ impl TraceUploadOptions<'_> {
if self.reload_config {
let _ = settings.reload();
}
- settings.get_agent_traces_enabled()
+ settings.errors().is_empty() && settings.get_agent_traces_enabled()
}
}
@@ -93,16 +93,33 @@ pub(super) async fn perform_agent_trace_upload(
if options.require_enabled && !options.enabled() {
return TraceUploadResult::Disabled;
}
- let body = match tokio::fs::read_to_string(session_file).await {
+ // Bound the read itself: an append racing the initial stat must not allocate
+ // an arbitrarily large body or bypass the server's 20 MiB limit.
+ let read_body = async {
+ use tokio::io::AsyncReadExt;
+ let file = tokio::fs::File::open(session_file).await?;
+ let mut body = String::new();
+ file.take(MAX_TRACE_BYTES + 1)
+ .read_to_string(&mut body)
+ .await?;
+ Ok::<_, std::io::Error>(body)
+ };
+ let body = match read_body.await {
Ok(body) => body,
Err(error) => {
return TraceUploadResult::Failed {
status_code: None,
message: error.to_string(),
retry_after_ms: None,
- }
+ };
}
};
+ if body.len() as u64 > MAX_TRACE_BYTES {
+ return TraceUploadResult::TooLarge {
+ size: body.len() as u64,
+ max_bytes: MAX_TRACE_BYTES,
+ };
+ }
if body.trim().is_empty() {
return TraceUploadResult::EmptySession;
}
@@ -151,7 +168,7 @@ pub(super) async fn perform_agent_trace_upload(
status_code: None,
message: error.message(),
retry_after_ms: None,
- }
+ };
}
};
if !(200..300).contains(&response.status) {
@@ -193,7 +210,7 @@ pub(super) async fn perform_agent_trace_upload(
/// The gate runs before every attempt, the retriable statuses/network
/// errors back off with jitter, and 503 honors `Retry-After`.
-async fn fetch_with_retry(
+pub(super) async fn fetch_with_retry(
options: &TraceUploadOptions<'_>,
url: &str,
headers: Vec<(String, String)>,
@@ -210,6 +227,9 @@ async fn fetch_with_retry(
if options.cancel.is_some_and(TraceUploadCancel::is_cancelled) {
return Err(TraceHttpError::Cancelled);
}
+ if options.require_enabled && !options.enabled() {
+ return Err(TraceHttpError::Cancelled);
+ }
let mut retry_delay_ms: Option<u64> = None;
let result = options
.http
diff --git a/crates/pa-core/src/platform/lock_dir.rs b/crates/pa-core/src/platform/lock_dir.rs
index 15c0977a83..c37ad39700 100644
--- a/crates/pa-core/src/platform/lock_dir.rs
+++ b/crates/pa-core/src/platform/lock_dir.rs
@@ -486,6 +486,30 @@ impl Drop for LockDir {
}
}
+/// Advisory exclusive lock on an open file, released on drop or process
+/// death — the platform-walled form of `std::fs::File::lock`. OS file locks
+/// (crash-released, no stale-lock reclaim) live here so platform-specific
+/// behavior stays auditable in one place, like the directory locks above.
+///
+/// # Errors
+///
+/// `WouldBlock` (blocking form: after interruption) never applies; other I/O
+/// errors as the platform reports them.
+pub fn lock_exclusive(file: &std::fs::File) -> io::Result<()> {
+ file.lock()
+}
+
+/// Nonblocking [`lock_exclusive`]: any held lock (fresh contention or I/O
+/// failure) surfaces as an error carrying the platform's try-lock error.
+///
+/// # Errors
+///
+/// The platform's [`std::fs::TryLockError`] (contention included) wrapped in
+/// an I/O error; nothing is returned on success.
+pub fn try_lock_exclusive(file: &std::fs::File) -> io::Result<()> {
+ file.try_lock().map_err(io::Error::other)
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/pa-core/src/platform/mod.rs b/crates/pa-core/src/platform/mod.rs
index 4cecd29760..1b0f6bda9c 100644
--- a/crates/pa-core/src/platform/mod.rs
+++ b/crates/pa-core/src/platform/mod.rs
@@ -10,7 +10,7 @@ pub mod process;
pub mod shell;
pub use fs::fsync;
-pub use lock_dir::LockDir;
+pub use lock_dir::{lock_exclusive, try_lock_exclusive, LockDir};
pub use perms::{
file_mode, is_executable, is_readable_writable, restrict_dir, restrict_file, set_private_mode,
};
diff --git a/crates/pa-core/src/session_engine/engine.rs b/crates/pa-core/src/session_engine/engine.rs
index b1123bfcb8..4d0fa4a45c 100644
--- a/crates/pa-core/src/session_engine/engine.rs
+++ b/crates/pa-core/src/session_engine/engine.rs
@@ -174,7 +174,7 @@ pub async fn create_session(mut config: SessionEngineConfig) -> anyhow::Result<S
let cwd = config.cwd.clone();
// Session persistence first: the conversation-log path and the resume
// context both come from the session manager.
- let session_manager = config
+ let mut session_manager = config
.session_manager
.unwrap_or_else(|| SessionManager::in_memory(&cwd));
let conversation_log = {
@@ -189,6 +189,22 @@ pub async fn create_session(mut config: SessionEngineConfig) -> anyhow::Result<S
.map(|path| path.display().to_string())
})
};
+ let (settings, trace_consent) =
+ crate::agent_traces::ContinuousTraceUpload::load_settings(&cwd, &config.agent_dir);
+ let traces = session_manager
+ .is_persisted()
+ .then(|| {
+ crate::agent_traces::ContinuousTraceUpload::install(
+ &cwd,
+ &config.agent_dir,
+ session_manager.get_session_file(),
+ trace_consent,
+ )
+ })
+ .flatten();
+ if let Some(traces) = traces {
+ session_manager.on_persist(Box::new(move |path| traces.persisted(path)));
+ }
let wiring = super::runtime_wiring::wire_session_runtime(
session_manager,
&config.agent_dir,
@@ -200,7 +216,6 @@ pub async fn create_session(mut config: SessionEngineConfig) -> anyhow::Result<S
config.cron_store.clone(),
);
- let settings = crate::settings::SettingsManager::create(&cwd, &config.agent_dir);
let service_tier_preference = settings.get_default_service_tier();
// Captured before `settings` moves into the resource loader: the
// compaction budget and the auto-refine gates.
diff --git a/crates/pa-daemon/src/branch_navigation.rs b/crates/pa-daemon/src/branch_navigation.rs
index a873a5c8a3..b8761e2197 100644
--- a/crates/pa-daemon/src/branch_navigation.rs
+++ b/crates/pa-daemon/src/branch_navigation.rs
@@ -390,6 +390,10 @@ impl TreeNavigation {
let file = session_dir
.join(crate::session_store::session_file_name(forked.session_id()));
forked.set_path(file);
+ forked.trace_upload = store
+ .trace_upload
+ .as_ref()
+ .and_then(|traces| traces.forked(&forked.path));
if let Some(lease) = &store.lease {
forked.lease =
Some(lease.acquire_target(&forked.path).map_err(|error| {
@@ -429,7 +433,7 @@ impl TreeNavigation {
/// `fork`'s swap phase (TS `buildAndApplyReplacement`): the store, the engine's session
/// file, and the rebuilt context move onto the prepared fork file; the teardown runs
/// between the prepare and this swap.
- pub(crate) async fn replace_with_fork(&self, forked: SessionFile) -> Result<(), String> {
+ pub(crate) async fn replace_with_fork(&self, mut forked: SessionFile) -> Result<(), String> {
let branch_entries = forked.branch_file_entries();
let new_path = forked.path.clone();
// Prime the fork store's usage fold before it enters the core: the
@@ -439,10 +443,30 @@ impl TreeNavigation {
let _ =
tokio::task::spawn_blocking(move || crate::session_store::read_session_info(&primed))
.await;
- let previous = {
+ let (previous, transferred) = {
let mut core = self.core.lock().unwrap();
- core.store.replace(forked)
+ // A navigating fork replaces its source: when the prepare could not
+ // admit a distinct installation (registry at capacity), the
+ // publication takes the predecessor's slot. No fallible work sits
+ // between the transfer and the store swap.
+ let mut transferred = None;
+ if forked.trace_upload.is_none() {
+ transferred = core
+ .store
+ .as_ref()
+ .and_then(|old| old.trace_upload.as_ref())
+ .and_then(|traces| {
+ traces.rebind(std::path::Path::new(&core.cwd), &forked.path)
+ });
+ forked.trace_upload.clone_from(&transferred);
+ }
+ (core.store.replace(forked), transferred)
};
+ if let Some(traces) = &transferred {
+ // The fork was rewritten before its controller existed; the
+ // pre-written file owes a delivery only after publication.
+ traces.persisted(&new_path);
+ }
// The old store's lease release flushes the window and info
// sidecars (megabytes for a large session): off the core lock
// and the runtime.
@@ -603,3 +627,133 @@ impl Worker {
}
}
}
+
+#[cfg(test)]
+mod trace_fork_tests {
+ use super::*;
+
+ #[tokio::test]
+ async fn below_capacity_fork_keeps_its_prepared_controller() {
+ let _env = crate::trace_test_env::lock_env();
+ let dir = tempfile::tempdir().unwrap();
+ let agent_dir = dir.path().join("agent");
+ let session_dir = agent_dir.join("sessions");
+ std::fs::create_dir_all(&session_dir).unwrap();
+ let mut settings = pa_core::settings::SettingsManager::create(dir.path(), &agent_dir);
+ settings.set_agent_traces_enabled(true).unwrap();
+ let old_path = session_dir.join("old.jsonl");
+ let (_, consent) =
+ pa_core::agent_traces::ContinuousTraceUpload::load_settings(dir.path(), &agent_dir);
+ let controller = pa_core::agent_traces::ContinuousTraceUpload::install(
+ dir.path(),
+ &agent_dir,
+ Some(&old_path),
+ consent,
+ )
+ .unwrap();
+ let mut store = SessionFile::create(dir.path().to_string_lossy().as_ref(), None, 0);
+ store.set_path(old_path.clone());
+ store.trace_upload = Some(controller.clone());
+ let core = Arc::new(Mutex::new(SessionCore::test_core(
+ Some(store),
+ dir.path().to_string_lossy().into_owned(),
+ )));
+ let navigation = TreeNavigation::new(
+ Arc::new(crate::engine::ScriptedEngine::default()),
+ core.clone(),
+ Arc::new(tokio::sync::Notify::new()),
+ );
+ // A spare registry slot admits the fork's distinct controller at prepare
+ // time; publication must keep it attached for later writes.
+ let (_, fork_consent) =
+ pa_core::agent_traces::ContinuousTraceUpload::load_settings(dir.path(), &agent_dir);
+ let prepared = pa_core::agent_traces::ContinuousTraceUpload::install(
+ dir.path(),
+ &agent_dir,
+ Some(&session_dir.join("fork.jsonl")),
+ fork_consent,
+ )
+ .unwrap();
+ let mut forked = SessionFile::create(dir.path().to_string_lossy().as_ref(), None, 0);
+ forked.set_path(session_dir.join("fork.jsonl"));
+ forked.trace_upload = Some(prepared.clone());
+ forked.rewrite().unwrap();
+ navigation.replace_with_fork(forked).await.unwrap();
+ let live = core.lock().unwrap();
+ let published = live.store.as_ref().unwrap();
+ let hook = published
+ .trace_upload
+ .as_ref()
+ .expect("the prepared hook must survive");
+ assert!(
+ Arc::ptr_eq(hook, &prepared),
+ "publication must keep the prepared controller attached"
+ );
+ drop(live);
+ let outbox = agent_dir.join("agent-traces-outbox");
+ let markers = std::fs::read_dir(&outbox)
+ .expect("the rewritten fork file owes a delivery")
+ .filter_map(std::result::Result::ok)
+ .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json"))
+ .count();
+ assert_eq!(markers, 1);
+ }
+
+ #[tokio::test]
+ async fn fork_at_capacity_publishes_with_the_predecessors_slot() {
+ let _env = crate::trace_test_env::lock_env();
+ let dir = tempfile::tempdir().unwrap();
+ let agent_dir = dir.path().join("agent");
+ let session_dir = agent_dir.join("sessions");
+ std::fs::create_dir_all(&session_dir).unwrap();
+ let mut settings = pa_core::settings::SettingsManager::create(dir.path(), &agent_dir);
+ settings.set_agent_traces_enabled(true).unwrap();
+ let old_path = session_dir.join("old.jsonl");
+ let (_, consent) =
+ pa_core::agent_traces::ContinuousTraceUpload::load_settings(dir.path(), &agent_dir);
+ let controller = pa_core::agent_traces::ContinuousTraceUpload::install(
+ dir.path(),
+ &agent_dir,
+ Some(&old_path),
+ consent,
+ )
+ .unwrap();
+ let mut store = SessionFile::create(dir.path().to_string_lossy().as_ref(), None, 0);
+ store.set_path(old_path.clone());
+ store.trace_upload = Some(controller.clone());
+ let core = Arc::new(Mutex::new(SessionCore::test_core(
+ Some(store),
+ dir.path().to_string_lossy().into_owned(),
+ )));
+ let navigation = TreeNavigation::new(
+ Arc::new(crate::engine::ScriptedEngine::default()),
+ core.clone(),
+ Arc::new(tokio::sync::Notify::new()),
+ );
+ // At capacity the prepare-time admission is rejected (the registry
+ // bound is pa-core's, covered by its bounded-admission tests); the
+ // unhooked prepared fork is that rejection's daemon-side shape.
+ let mut forked = SessionFile::create(dir.path().to_string_lossy().as_ref(), None, 0);
+ forked.set_path(session_dir.join("fork.jsonl"));
+ forked.rewrite().unwrap();
+ navigation.replace_with_fork(forked).await.unwrap();
+ let live = core.lock().unwrap();
+ let published = live.store.as_ref().unwrap();
+ assert!(
+ published.trace_upload.is_some(),
+ "a replacing fork must publish with its predecessor's slot"
+ );
+ drop(live);
+ let outbox = agent_dir.join("agent-traces-outbox");
+ let markers: Vec<_> = std::fs::read_dir(&outbox)
+ .expect("the pre-written fork file owes a delivery")
+ .map(|entry| entry.expect("outbox entry").path())
+ .filter(|path| path.extension().is_some_and(|ext| ext == "json"))
+ .collect();
+ assert_eq!(
+ markers.len(),
+ 1,
+ "exactly the fork's pending marker must be recorded"
+ );
+ }
+}
diff --git a/crates/pa-daemon/src/lib.rs b/crates/pa-daemon/src/lib.rs
index f45c48d13c..05686a3c22 100644
--- a/crates/pa-daemon/src/lib.rs
+++ b/crates/pa-daemon/src/lib.rs
@@ -126,3 +126,5 @@ pub(crate) mod worker_stderr;
#[cfg(test)]
pub(crate) mod test_support;
+#[cfg(test)]
+pub(crate) mod trace_test_env;
diff --git a/crates/pa-daemon/src/session_navigation.rs b/crates/pa-daemon/src/session_navigation.rs
index 59c2bd217e..965c071c3a 100644
--- a/crates/pa-daemon/src/session_navigation.rs
+++ b/crates/pa-daemon/src/session_navigation.rs
@@ -17,6 +17,8 @@ use crate::worker::{SessionCore, Worker};
pub(crate) struct PreparedReplacement {
pub(crate) file: SessionFile,
pub(crate) cwd: Option<String>,
+ // A successful fresh write owes scheduling only after publication.
+ trace_persisted: bool,
}
/// The navigation surface: the prepare and swap phases of the replacement
@@ -35,7 +37,12 @@ impl SessionNavigation {
/// Swap the live session onto `file` (store, engine session file, rebuilt
/// context). The caller retires the previous runtime and rebinds the cwd
/// first, so the context park lands on the fresh session.
- async fn replace_session(&self, file: SessionFile) -> Result<(), String> {
+ async fn replace_session(&self, target: PreparedReplacement) -> Result<(), String> {
+ let PreparedReplacement {
+ mut file,
+ trace_persisted,
+ ..
+ } = target;
let branch_entries = file.branch_file_entries();
let new_path = file.path.clone();
// Prime the new store's usage fold before it enters the core: the
@@ -44,10 +51,24 @@ impl SessionNavigation {
let _ =
tokio::task::spawn_blocking(move || crate::session_store::read_session_info(&primed))
.await;
- let previous = {
+ let (previous, traces) = {
let mut core = self.core.lock().unwrap();
- core.store.replace(file)
+ if file.trace_upload.is_none() {
+ file.trace_upload = core
+ .store
+ .as_ref()
+ .and_then(|old| old.trace_upload.as_ref())
+ .and_then(|traces| traces.rebind(std::path::Path::new(&core.cwd), &file.path));
+ }
+ let traces = file.trace_upload.clone();
+ // No await or fallible preparation between slot transfer and publication.
+ (core.store.replace(file), traces)
};
+ if let Some(traces) = traces.filter(|_| trace_persisted) {
+ // The successful prepare wrote this file before its controller existed.
+ // Record bounded intent outside the core lock, after publication.
+ traces.persisted(&new_path);
+ }
// The old store's lease release flushes the window and info
// sidecars (megabytes for a large session): off the core lock
// and the runtime.
@@ -131,9 +152,11 @@ impl SessionNavigation {
}
}
// TS `newSession` keeps the runtime's cwd: the fresh session runs where the live one did.
+ let trace_persisted = !fresh.path.as_os_str().is_empty();
Ok(PreparedReplacement {
file: fresh,
cwd: None,
+ trace_persisted,
})
}
@@ -274,7 +297,11 @@ impl SessionNavigation {
));
}
}
- Ok(PreparedReplacement { file, cwd })
+ Ok(PreparedReplacement {
+ file,
+ cwd,
+ trace_persisted: false,
+ })
}
}
@@ -316,7 +343,7 @@ impl Worker {
if let Some(cwd) = target.cwd.as_deref() {
self.rebind_worker_cwd(cwd);
}
- match self.navigation.replace_session(target.file).await {
+ match self.navigation.replace_session(target).await {
Ok(()) => {
self.refresh_replaced_session_state().await;
self.reseed_service_tier_for_replacement();
@@ -870,3 +897,51 @@ mod tests {
}
}
}
+
+#[cfg(test)]
+mod trace_replacement_tests {
+ use super::*;
+
+ #[test]
+ fn failed_new_session_prepare_preserves_the_live_trace_registration() {
+ let _env = crate::trace_test_env::lock_env();
+ let dir = tempfile::tempdir().unwrap();
+ let blocked = dir.path().join("not-a-directory");
+ std::fs::write(&blocked, "synthetic blocker").unwrap();
+ let old_path = blocked.join("old.jsonl");
+ let agent_dir = dir.path().join("agent");
+ let (_, consent) =
+ pa_core::agent_traces::ContinuousTraceUpload::load_settings(dir.path(), &agent_dir);
+ let controller = pa_core::agent_traces::ContinuousTraceUpload::install(
+ dir.path(),
+ &agent_dir,
+ Some(&old_path),
+ consent,
+ )
+ .unwrap();
+ let mut store = SessionFile::create(dir.path().to_str().unwrap(), None, 0);
+ store.set_path(old_path.clone());
+ store.trace_upload = Some(controller.clone());
+ let core = Arc::new(Mutex::new(SessionCore::test_core(
+ Some(store),
+ dir.path().to_string_lossy().into_owned(),
+ )));
+ let navigation = SessionNavigation::new(
+ Arc::new(crate::engine::ScriptedEngine::default()),
+ core.clone(),
+ );
+ assert!(navigation.prepare_new_session(&json!({})).is_err());
+ let live = core.lock().unwrap();
+ assert_eq!(live.store.as_ref().unwrap().path, old_path);
+ assert!(Arc::ptr_eq(
+ live.store.as_ref().unwrap().trace_upload.as_ref().unwrap(),
+ &controller
+ ));
+ drop(live);
+ // A cancelled predecessor cannot transfer its registry slot again.
+ assert!(controller
+ .rebind(dir.path(), &dir.path().join("probe.jsonl"))
+ .is_some());
+ assert!(!agent_dir.join("agent-traces-outbox").exists());
+ }
+}
diff --git a/crates/pa-daemon/src/session_store.rs b/crates/pa-daemon/src/session_store.rs
index 6c3e0e7ee1..d0197d818a 100644
--- a/crates/pa-daemon/src/session_store.rs
+++ b/crates/pa-daemon/src/session_store.rs
@@ -122,6 +122,7 @@ pub struct SessionFile {
pub(crate) by_id: HashMap<String, usize>,
pub(crate) leaf_id: Option<String>,
pub(crate) window: Option<SessionWindow>,
+ pub(crate) trace_upload: Option<std::sync::Arc<pa_core::agent_traces::ContinuousTraceUpload>>,
pub(crate) lease: Option<std::sync::Arc<crate::lease::SessionLease>>,
/// Rows the full reader skipped as unparsable (the TS loader's
/// lenient skip): zero on a healthy file. A reopen that found damage
diff --git a/crates/pa-daemon/src/session_store/read.rs b/crates/pa-daemon/src/session_store/read.rs
index 6e2e4032b0..246513d7ba 100644
--- a/crates/pa-daemon/src/session_store/read.rs
+++ b/crates/pa-daemon/src/session_store/read.rs
@@ -156,6 +156,7 @@ impl SessionFile {
leaf_id: None,
window: None,
lease: None,
+ trace_upload: None,
anthropic_warning_shown: false,
skipped_lines: 0,
};
@@ -214,6 +215,7 @@ impl SessionFile {
leaf_id: None,
window: None,
lease: None,
+ trace_upload: None,
anthropic_warning_shown,
skipped_lines: 0,
};
@@ -281,6 +283,7 @@ impl SessionFile {
}
full.leaf_id.clone_from(&self.leaf_id);
full.lease.clone_from(&self.lease);
+ full.trace_upload.clone_from(&self.trace_upload);
// The merged store's leaf is the window's leaf: re-derive the gate
// from the merged ACTIVE branch — the full open's own-leaf answer
// can disagree, a post-snapshot marker must hydrate, and an
@@ -309,6 +312,7 @@ impl SessionFile {
leaf_id: None,
window: None,
lease: None,
+ trace_upload: None,
anthropic_warning_shown: false,
skipped_lines: 0,
}
diff --git a/crates/pa-daemon/src/session_store/tests.rs b/crates/pa-daemon/src/session_store/tests.rs
index e719f3d53b..5a971e88e4 100644
--- a/crates/pa-daemon/src/session_store/tests.rs
+++ b/crates/pa-daemon/src/session_store/tests.rs
@@ -1262,3 +1262,27 @@ fn session_header_line_leads_with_the_type_tag() {
let line = serde_json::to_string(&session_header_line(&header)).unwrap();
assert!(line.starts_with("{\"type\":\"session\",\"version\":3,\"id\":\"abc\""));
}
+
+#[cfg(unix)] // Windows uses the full reader rather than windowed hydration.
+#[test]
+fn hydrating_full_history_preserves_the_trace_controller() {
+ let dir = temp_dir();
+ let path = dir.join("trace-hydration.jsonl");
+ let mut original = SessionFile::create(dir.to_str().unwrap(), None, 0);
+ original.append_message(&json!({"role":"user", "content":"synthetic", "timestamp":1u64}));
+ original.set_path(path.clone());
+ original.rewrite().unwrap();
+ let mut windowed = SessionFile::open_windowed(&path).unwrap();
+ assert!(windowed.window.is_some());
+ let (_, consent) = pa_core::agent_traces::ContinuousTraceUpload::load_settings(&dir, &dir);
+ let controller =
+ pa_core::agent_traces::ContinuousTraceUpload::install(&dir, &dir, Some(&path), consent)
+ .unwrap();
+ windowed.trace_upload = Some(controller.clone());
+ windowed.ensure_full_history().unwrap();
+ assert!(windowed.window.is_none());
+ assert!(std::sync::Arc::ptr_eq(
+ windowed.trace_upload.as_ref().unwrap(),
+ &controller
+ ));
+}
diff --git a/crates/pa-daemon/src/session_store/write.rs b/crates/pa-daemon/src/session_store/write.rs
index ffb580de94..9692197a3a 100644
--- a/crates/pa-daemon/src/session_store/write.rs
+++ b/crates/pa-daemon/src/session_store/write.rs
@@ -40,6 +40,9 @@ impl SessionFile {
pa_core::session::window::AppendOwnership::Unleased,
)?,
}
+ if let Some(traces) = &self.trace_upload {
+ traces.persisted(&self.path);
+ }
Ok(())
}
@@ -128,6 +131,9 @@ impl SessionFile {
}
pa_core::platform::rename_onto(&temp, path)
.with_context(|| format!("persist {}", path.display()))?;
+ if let Some(traces) = &self.trace_upload {
+ traces.persisted(&self.path);
+ }
Ok(())
}
@@ -216,6 +222,9 @@ impl SessionFile {
// The line is in the file now: index it so the in-memory leaf matches what a
// reload sees.
self.push_index(entry);
+ if let Some(traces) = &self.trace_upload {
+ traces.persisted(&self.path);
+ }
} else {
// The rewrite path serializes the whole index, so the entry must be
// indexed first; a failed rewrite rolls the index back. The live
diff --git a/crates/pa-daemon/src/session_tree.rs b/crates/pa-daemon/src/session_tree.rs
index f631008f53..a74568b828 100644
--- a/crates/pa-daemon/src/session_tree.rs
+++ b/crates/pa-daemon/src/session_tree.rs
@@ -245,6 +245,10 @@ impl SessionFile {
// Same directory as the source session, like TS `createUniqueSessionFileTarget`.
let file = session_dir.join(session_file_name(forked.session_id()));
forked.set_path(file);
+ forked.trace_upload = self
+ .trace_upload
+ .as_ref()
+ .and_then(|traces| traces.forked(&forked.path));
if let Some(lease) = &self.lease {
forked.lease = Some(lease.acquire_target(&forked.path)?);
}
diff --git a/crates/pa-daemon/src/trace_test_env.rs b/crates/pa-daemon/src/trace_test_env.rs
new file mode 100644
index 0000000000..bc619a3a5b
--- /dev/null
+++ b/crates/pa-daemon/src/trace_test_env.rs
@@ -0,0 +1,45 @@
+//! Test-only: the trace uploader resolves credentials from process env
+//! (`PRIME_AGENT_TRACES_API_KEY`, then `PRIME_API_KEY`), so daemon tests
+//! that install consent-enabled controllers must clear both and serialize
+//! with each other while the values stay stable.
+
+use std::sync::MutexGuard;
+
+pub(crate) struct TraceEnv {
+ // Held for the guard's lifetime; the lock's value is never read.
+ #[allow(dead_code)]
+ lock: MutexGuard<'static, ()>,
+ saved: [(&'static str, Option<std::ffi::OsString>); 2],
+}
+
+pub(crate) fn lock_env() -> TraceEnv {
+ let lock = env_lock();
+ let saved = [
+ (
+ "PRIME_AGENT_TRACES_API_KEY",
+ std::env::var_os("PRIME_AGENT_TRACES_API_KEY"),
+ ),
+ ("PRIME_API_KEY", std::env::var_os("PRIME_API_KEY")),
+ ];
+ std::env::remove_var("PRIME_AGENT_TRACES_API_KEY");
+ std::env::remove_var("PRIME_API_KEY");
+ TraceEnv { lock, saved }
+}
+
+impl Drop for TraceEnv {
+ fn drop(&mut self) {
+ for (name, value) in &self.saved {
+ if let Some(value) = value {
+ std::env::set_var(name, value);
+ } else {
+ std::env::remove_var(name);
+ }
+ }
+ }
+}
+
+fn env_lock() -> MutexGuard<'static, ()> {
+ static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+ LOCK.lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+}
diff --git a/crates/pa-daemon/src/worker/create.rs b/crates/pa-daemon/src/worker/create.rs
index 6649e716e6..44f4a1305c 100644
--- a/crates/pa-daemon/src/worker/create.rs
+++ b/crates/pa-daemon/src/worker/create.rs
@@ -61,7 +61,9 @@ impl Worker {
return response_failure(
None,
"create",
- &format!("Invalid thinking level \"{level}\". Valid values: off, minimal, low, medium, high, xhigh, max"),
+ &format!(
+ "Invalid thinking level \"{level}\". Valid values: off, minimal, low, medium, high, xhigh, max"
+ ),
None,
);
}
@@ -197,6 +199,18 @@ impl Worker {
// The fresh arms defer their creation prefix to after the startup scope
// registers, so a fresh `--models` session persists the scoped startup pick.
let mut fresh_prefix = FreshPrefixPlan::None;
+ let (settings, trace_consent) = pa_core::agent_traces::ContinuousTraceUpload::load_settings(
+ std::path::Path::new(&cwd),
+ &self.config.agent_dir,
+ );
+ let traces = |path: &std::path::Path| {
+ pa_core::agent_traces::ContinuousTraceUpload::install(
+ std::path::Path::new(&cwd),
+ &self.config.agent_dir,
+ Some(path),
+ trace_consent.clone(),
+ )
+ };
let mut store = match (&session_path, no_session) {
(Some(path), false) if path.exists() => {
let loaded = {
@@ -215,6 +229,7 @@ impl Worker {
};
match loaded {
Ok(mut opened) => {
+ opened.trace_upload = traces(&opened.path);
opened_existing_session = true;
if opened.skipped_lines > 0 {
// The rows stay on disk (the append-only
@@ -298,6 +313,7 @@ impl Worker {
rlm_depth.unwrap_or(0),
);
created.set_path(path.clone());
+ created.trace_upload = traces(&created.path);
let acquired = {
let path = path.clone();
let agent_dir = self.config.agent_dir.clone();
@@ -336,6 +352,7 @@ impl Worker {
);
let path = session_dir.join(session_file_name(created.session_id()));
created.set_path(path.clone());
+ created.trace_upload = traces(&created.path);
let acquired = {
let path = path.clone();
let agent_dir = self.config.agent_dir.clone();
@@ -362,7 +379,7 @@ impl Worker {
"create",
"Session cannot be both no-session and session-pathed",
None,
- )
+ );
}
};
@@ -468,7 +485,6 @@ impl Worker {
// The session's settings-seeded switches: a restarted session re-seeds
// its auto-compaction flag from the persisted `compaction.enabled`.
let (service_tier, steering_mode, follow_up_mode, auto_compaction_enabled) = {
- let settings = pa_core::settings::SettingsManager::create(&cwd, &self.config.agent_dir);
let queue_mode = |mode: pa_core::settings::QueueModeSetting| -> String {
match mode {
pa_core::settings::QueueModeSetting::All => "all".to_string(),