Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(redact): back off cloud failures (#5070)

Louis Beaumont08952eba5043
diff --git a/crates/screenpipe-redact/src/pipeline.rs b/crates/screenpipe-redact/src/pipeline.rs
index 4d8b329cd..0124db5e5 100644
--- a/crates/screenpipe-redact/src/pipeline.rs
+++ b/crates/screenpipe-redact/src/pipeline.rs
@@ -23,9 +23,10 @@
//! Substring search is fine here because we already know the AI
//! produced a replacement — we just need offsets for the audit trail.
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
use async_trait::async_trait;
+use tokio::{sync::Mutex, time::Instant};
use crate::{
adapters::regex::{self as regex_adapter, RegexRedactor},
@@ -59,6 +60,87 @@ impl Default for PipelineConfig {
}
}
+/// Keep a failed cloud backend off the per-text hot path long enough for
+/// transient DNS / transport failures to settle. A fixed, bounded cooldown
+/// also caps persistent-outage probes (and their warnings) at one per minute.
+const AI_FAILURE_COOLDOWN: Duration = Duration::from_secs(60);
+
+#[derive(Clone, Copy)]
+struct AiAttempt {
+ generation: u64,
+ probe: bool,
+}
+
+#[derive(Default)]
+struct AiCircuitState {
+ generation: u64,
+ retry_at: Option<Instant>,
+ probe_in_flight: bool,
+}
+
+/// Thread-safe failure circuit for the remote AI fallback. The mutex is held
+/// only while updating this tiny state machine, never across an AI request.
+struct AiFailureCircuit {
+ state: Mutex<AiCircuitState>,
+}
+
+impl AiFailureCircuit {
+ fn new() -> Self {
+ Self {
+ state: Mutex::new(AiCircuitState::default()),
+ }
+ }
+
+ /// Return a permit for a normal request, or for the sole recovery probe
+ /// once the cooldown has elapsed. `None` means use regex-only for now.
+ async fn begin_attempt(&self) -> Option<AiAttempt> {
+ let mut state = self.state.lock().await;
+ let probe = match state.retry_at {
+ None => false,
+ Some(retry_at) if Instant::now() < retry_at => return None,
+ Some(_) if state.probe_in_flight => return None,
+ Some(_) => {
+ state.probe_in_flight = true;
+ true
+ }
+ };
+ Some(AiAttempt {
+ generation: state.generation,
+ probe,
+ })
+ }
+
+ /// Open (or re-open after a failed probe) the circuit. Generation checks
+ /// prevent older concurrent requests from extending a newer cooldown or
+ /// producing duplicate warnings.
+ async fn record_failure(&self, attempt: AiAttempt) -> bool {
+ let mut state = self.state.lock().await;
+ if attempt.generation != state.generation {
+ return false;
+ }
+ state.generation = state.generation.wrapping_add(1);
+ state.retry_at = Some(Instant::now() + AI_FAILURE_COOLDOWN);
+ state.probe_in_flight = false;
+ true
+ }
+
+ /// A successful recovery probe closes the circuit. Normal successes while
+ /// the circuit is already closed require no state change.
+ async fn record_success(&self, attempt: AiAttempt) -> bool {
+ if !attempt.probe {
+ return false;
+ }
+ let mut state = self.state.lock().await;
+ if attempt.generation != state.generation {
+ return false;
+ }
+ state.generation = state.generation.wrapping_add(1);
+ state.retry_at = None;
+ state.probe_in_flight = false;
+ true
+ }
+}
+
/// Drop spans whose label isn't in the policy, then rebuild `redacted`
/// from `input` using only the surviving spans' replacements. Caller
/// must already have `spans` anchored to `input`. When `pseudonyms` is
@@ -127,6 +209,9 @@ pub struct Pipeline {
ai: Option<Arc<dyn Redactor>>,
cfg: PipelineConfig,
cache: RedactionCache,
+ /// Only remote Tinfoil traffic needs outage backoff. Local adapters keep
+ /// their existing behavior and never pay this state check.
+ ai_failure_circuit: Option<AiFailureCircuit>,
/// When set, redacted spans render as stable per-install pseudonym
/// tokens instead of static `[LABEL]` placeholders (issue #4206,
/// opt-in). `None` keeps the historic behavior.
@@ -155,17 +240,20 @@ impl Pipeline {
..Default::default()
},
cache: RedactionCache::with_defaults(),
+ ai_failure_circuit: None,
pseudonyms: None,
}
}
/// Pipeline with regex + a configured AI fallback.
pub fn regex_then_ai(ai: Arc<dyn Redactor>, cfg: PipelineConfig) -> Self {
+ let ai_failure_circuit = (ai.name() == "tinfoil").then(AiFailureCircuit::new);
Self {
regex: RegexRedactor::new(),
ai: Some(ai),
cfg,
cache: RedactionCache::with_defaults(),
+ ai_failure_circuit,
pseudonyms: None,
}
}
@@ -233,10 +321,36 @@ impl Redactor for Pipeline {
&& current.input.chars().count() >= self.cfg.ai_min_chars
&& current.spans.len() < self.cfg.ai_skip_if_regex_spans;
+ // Regex-only fallbacks caused by a transient AI outage must not be
+ // cached for the normal one-hour TTL. That lets the same input get
+ // the stronger AI pass after the circuit recovers; successful AI
+ // results and intentional regex-only results remain cached.
+ let mut cache_output = true;
+
if want_ai {
let ai = self.ai.as_ref().expect("checked above");
+ let attempt = match &self.ai_failure_circuit {
+ Some(circuit) => circuit.begin_attempt().await.map(Some),
+ None => Some(None),
+ };
+
+ let Some(circuit_attempt) = attempt else {
+ out.push(current);
+ continue;
+ };
+
match ai.redact(&current.redacted).await {
Ok(ai_out) => {
+ if let (Some(circuit), Some(attempt)) =
+ (&self.ai_failure_circuit, circuit_attempt)
+ {
+ if circuit.record_success(attempt).await {
+ tracing::info!(
+ backend = ai.name(),
+ "AI redactor recovered; closing failure circuit"
+ );
+ }
+ }
let redacted = if ai_out.spans.is_empty() {
// Span-less adapter (the Tinfoil enclave
// returns redacted text only, no spans). It
@@ -265,9 +379,21 @@ impl Redactor for Pipeline {
};
}
Err(RedactError::Unavailable(_)) => {
- // AI not available — keep regex output.
+ // AI not available — keep regex output and avoid
+ // retrying it for every text in this outage window.
+ cache_output = self.ai_failure_circuit.is_none();
+ if let (Some(circuit), Some(attempt)) =
+ (&self.ai_failure_circuit, circuit_attempt)
+ {
+ circuit.record_failure(attempt).await;
+ }
}
Err(e) => {
+ cache_output = self.ai_failure_circuit.is_none();
+ let circuit_opened = match (&self.ai_failure_circuit, circuit_attempt) {
+ (Some(circuit), Some(attempt)) => circuit.record_failure(attempt).await,
+ _ => true,
+ };
// Transient failure — log via tracing, keep
// regex output. We do NOT fail the whole
// batch because that would block the
@@ -275,23 +401,31 @@ impl Redactor for Pipeline {
// chain so we surface the underlying cause
// (TLS handshake / DNS / refused / etc.)
// rather than just the catchall wrapper.
- let mut detail = format!("{}", e);
- let mut src: Option<&dyn std::error::Error> = std::error::Error::source(&e);
- while let Some(s) = src {
- detail.push_str(" → ");
- detail.push_str(&s.to_string());
- src = s.source();
+ // A cloud outage logs once when the circuit opens;
+ // stale concurrent requests are intentionally quiet.
+ if circuit_opened {
+ let mut detail = format!("{}", e);
+ let mut src: Option<&dyn std::error::Error> =
+ std::error::Error::source(&e);
+ while let Some(s) = src {
+ detail.push_str(" → ");
+ detail.push_str(&s.to_string());
+ src = s.source();
+ }
+ tracing::warn!(
+ error = %e,
+ detail = %detail,
+ cooldown_seconds = AI_FAILURE_COOLDOWN.as_secs(),
+ "AI redactor failed; falling back to regex-only output"
+ );
}
- tracing::warn!(
- error = %e,
- detail = %detail,
- "AI redactor failed; falling back to regex-only output"
- );
}
}
}
- self.cache.insert(key, current.clone()).await;
+ if cache_output {
+ self.cache.insert(key, current.clone()).await;
+ }
out.push(current);
}
@@ -341,6 +475,57 @@ mod tests {
}
}
+ /// Cloud-shaped test redactor that fails a configured number of calls,
+ /// then returns a visible successful result.
+ struct FlakyAi {
+ calls: AtomicUsize,
+ failures_remaining: AtomicUsize,
+ }
+
+ impl FlakyAi {
+ fn new(failures: usize) -> Self {
+ Self {
+ calls: AtomicUsize::new(0),
+ failures_remaining: AtomicUsize::new(failures),
+ }
+ }
+ }
+
+ #[async_trait]
+ impl Redactor for FlakyAi {
+ fn name(&self) -> &str {
+ "tinfoil"
+ }
+
+ fn version(&self) -> u32 {
+ 43
+ }
+
+ async fn redact_batch(
+ &self,
+ texts: &[String],
+ ) -> Result<Vec<RedactionOutput>, RedactError> {
+ self.calls.fetch_add(1, Ordering::SeqCst);
+ let should_fail = self
+ .failures_remaining
+ .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| {
+ remaining.checked_sub(1)
+ })
+ .is_ok();
+ if should_fail {
+ return Err(RedactError::Runtime("temporary DNS failure".into()));
+ }
+ Ok(texts
+ .iter()
+ .map(|text| RedactionOutput {
+ input: text.clone(),
+ redacted: text.to_uppercase(),
+ spans: vec![],
+ })
+ .collect())
+ }
+ }
+
#[async_trait]
impl Redactor for UppercaseAi {
fn name(&self) -> &str {
@@ -424,6 +609,87 @@ mod tests {
assert_eq!(ai.calls.load(Ordering::SeqCst), 1);
}
+ #[tokio::test]
+ async fn cloud_failure_skips_rest_of_batch_during_cooldown() {
+ let ai = Arc::new(FlakyAi::new(usize::MAX));
+ let p = Pipeline::regex_then_ai(ai.clone(), PipelineConfig::default());
+ let texts = vec![
+ "first unique text long enough for cloud redaction".to_string(),
+ "second unique text long enough for cloud redaction".to_string(),
+ "third unique text long enough for cloud redaction".to_string(),
+ ];
+
+ let outputs = p.redact_batch(&texts).await.unwrap();
+
+ assert_eq!(ai.calls.load(Ordering::SeqCst), 1);
+ assert_eq!(outputs.len(), texts.len());
+ assert!(outputs
+ .iter()
+ .zip(&texts)
+ .all(|(output, input)| output.redacted == *input));
+ }
+
+ #[tokio::test]
+ async fn cloud_circuit_is_shared_across_concurrent_callers() {
+ let ai = Arc::new(FlakyAi::new(usize::MAX));
+ let p = Arc::new(Pipeline::regex_then_ai(
+ ai.clone(),
+ PipelineConfig::default(),
+ ));
+
+ p.redact("initial unique text long enough to open the cloud circuit")
+ .await
+ .unwrap();
+ assert_eq!(ai.calls.load(Ordering::SeqCst), 1);
+
+ let mut tasks = tokio::task::JoinSet::new();
+ for i in 0..32 {
+ let p = Arc::clone(&p);
+ tasks.spawn(async move {
+ p.redact(&format!(
+ "concurrent unique text number {i} long enough for redaction"
+ ))
+ .await
+ });
+ }
+ while let Some(result) = tasks.join_next().await {
+ result.unwrap().unwrap();
+ }
+
+ assert_eq!(
+ ai.calls.load(Ordering::SeqCst),
+ 1,
+ "open circuit must suppress every concurrent cloud request"
+ );
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn cloud_circuit_recovers_and_only_caches_success() {
+ let ai = Arc::new(FlakyAi::new(1));
+ let p = Pipeline::regex_then_ai(ai.clone(), PipelineConfig::default());
+ let text = "same text remains eligible after a temporary cloud outage";
+
+ let degraded = p.redact(text).await.unwrap();
+ assert_eq!(degraded.redacted, text);
+ assert_eq!(ai.calls.load(Ordering::SeqCst), 1);
+
+ // A degraded regex-only result is deliberately not cached, but the
+ // open circuit still keeps this repeat off the network.
+ let during_cooldown = p.redact(text).await.unwrap();
+ assert_eq!(during_cooldown.redacted, text);
+ assert_eq!(ai.calls.load(Ordering::SeqCst), 1);
+
+ tokio::time::advance(AI_FAILURE_COOLDOWN).await;
+ let recovered = p.redact(text).await.unwrap();
+ assert_eq!(recovered.redacted, text.to_uppercase());
+ assert_eq!(ai.calls.load(Ordering::SeqCst), 2);
+
+ // Successful output retains the existing long-lived cache behavior.
+ let cached = p.redact(text).await.unwrap();
+ assert_eq!(cached, recovered);
+ assert_eq!(ai.calls.load(Ordering::SeqCst), 2);
+ }
+
#[tokio::test]
async fn ai_skipped_when_input_short() {
let ai = Arc::new(UppercaseAi::new());