YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(connections,pipes): prevent silent data loss on persistence paths (#4120)
Louis Beaumont1ef6f6cae04b
diff --git a/crates/screenpipe-core/src/connections/sync.rs b/crates/screenpipe-core/src/connections/sync.rsindex 722640b83..626fd8352 100644--- a/crates/screenpipe-core/src/connections/sync.rs+++ b/crates/screenpipe-core/src/connections/sync.rs@@ -711,10 +711,7 @@ pub async fn apply_manifest_to_disk(if let Err(e) = store.set_json(&secret_key, token).await {errors.push(format!("failed to save oauth token {}: {}", key, e));}- } else if let Err(e) = std::fs::write(- connection_oauth_file_path(screenpipe_dir, key),- serde_json::to_string_pretty(token).unwrap_or_default(),- ) {+ } else if let Err(e) = write_oauth_token_file(screenpipe_dir, key, token) {errors.push(format!("failed to write oauth token {}: {}", key, e));}@@ -809,10 +806,7 @@ pub async fn apply_manifest_to_disk(errors.push(format!("missing oauth token for {}", key));continue;};- if let Err(e) = std::fs::write(- connection_oauth_file_path(screenpipe_dir, key),- serde_json::to_string_pretty(token).unwrap_or_default(),- ) {+ if let Err(e) = write_oauth_token_file(screenpipe_dir, key, token) {errors.push(format!("failed to write oauth token {}: {}", key, e));}connection_file.remove(key);@@ -852,13 +846,39 @@ pub async fn apply_manifest_to_disk(errors}+/// Atomic write of a per-connection OAuth token file (tmp + rename), matching+/// write_connection_file. The previous inline `std::fs::write(..).+/// unwrap_or_default()` was non-atomic (a crash mid-write truncates the token)+/// and would silently persist an empty `""` token on a serialize failure; here+/// the serialize error is surfaced and the old token survives a failed write.+fn write_oauth_token_file(+ screenpipe_dir: &Path,+ manifest_key: &str,+ token: &Value,+) -> Result<(), String> {+ let path = connection_oauth_file_path(screenpipe_dir, manifest_key);+ let tmp = path.with_extension("tmp");+ let json = serde_json::to_string_pretty(token).map_err(|e| format!("serialize: {e}"))?;+ std::fs::write(&tmp, &json).map_err(|e| format!("write tmp {:?}: {}", tmp, e))?;+ std::fs::rename(&tmp, &path).map_err(|e| format!("rename {:?}: {}", path, e))?;+ Ok(())+}++/// Atomic write of connections.json (tmp file + rename), matching+/// write_connection_tombstones. A direct write could truncate the file if it+/// failed partway (crash, disk full), wiping every saved connection/credential;+/// the temp-then-rename keeps the previous good file intact until the new one+/// is fully written.fn write_connection_file(screenpipe_dir: &Path,connections: &HashMap<String, SavedConnection>,) -> Result<(), String> {let path = screenpipe_dir.join("connections.json");+ let tmp = path.with_extension("tmp");let json = serde_json::to_string_pretty(connections).map_err(|e| format!("serialize: {e}"))?;- std::fs::write(&path, &json).map_err(|e| format!("write {}: {}", path.display(), e))+ std::fs::write(&tmp, &json).map_err(|e| format!("write tmp {:?}: {}", tmp, e))?;+ std::fs::rename(&tmp, &path).map_err(|e| format!("rename {:?}: {}", path, e))?;+ Ok(())}// ---------------------------------------------------------------------------@@ -1164,4 +1184,94 @@ mod tests {assert_eq!(manifest.connections["notion"].integration_id, "notion");assert!(!manifest.connections["notion"].is_oauth);}++ #[test]+ fn write_connection_file_preserves_existing_on_failed_write() {+ // A failed write must NOT destroy the previously-saved connections.+ // Force the temp write to fail by occupying the temp path with a+ // directory; the atomic writer then errors and leaves connections.json+ // untouched. A non-atomic direct write would clobber it instead — so+ // this test fails against the old implementation and passes against the+ // atomic one.+ let dir = TempDir::new().unwrap();++ let mut good = HashMap::new();+ good.insert(+ "gmail".to_string(),+ SavedConnection {+ enabled: true,+ credentials: Map::new(),+ },+ );+ write_connection_file(dir.path(), &good).expect("initial write");+ assert!(read_connection_file(dir.path()).contains_key("gmail"));++ // Block the temp path so the replacement write cannot land.+ fs::create_dir(dir.path().join("connections.tmp")).unwrap();++ let mut replacement = HashMap::new();+ replacement.insert(+ "outlook".to_string(),+ SavedConnection {+ enabled: true,+ credentials: Map::new(),+ },+ );+ let result = write_connection_file(dir.path(), &replacement);++ assert!(result.is_err(), "write into a blocked temp path must error");+ let reloaded = read_connection_file(dir.path());+ assert!(+ reloaded.contains_key("gmail"),+ "existing connections must survive a failed write"+ );+ assert!(!reloaded.contains_key("outlook"));+ }++ #[test]+ fn write_oauth_token_file_preserves_existing_on_failed_write() {+ // A failed token write must not destroy the existing token. Block the+ // temp path with a directory to force failure, then assert the previous+ // token is intact and the call errors. Fails against the old inline+ // direct write (which would overwrite); passes against the atomic one.+ let dir = TempDir::new().unwrap();+ let good = serde_json::json!({ "access_token": "good" });+ write_oauth_token_file(dir.path(), "gmail", &good).expect("initial write");++ let path = connection_oauth_file_path(dir.path(), "gmail");+ assert!(path.exists());++ fs::create_dir(path.with_extension("tmp")).unwrap();++ let replacement = serde_json::json!({ "access_token": "new" });+ let result = write_oauth_token_file(dir.path(), "gmail", &replacement);++ assert!(result.is_err(), "write into a blocked temp path must error");+ let reloaded: Value = serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap();+ assert_eq!(+ reloaded["access_token"], "good",+ "existing token must survive a failed write"+ );+ }++ #[test]+ fn write_connection_file_roundtrips_and_leaves_no_temp() {+ let dir = TempDir::new().unwrap();+ let mut conns = HashMap::new();+ conns.insert(+ "slack".to_string(),+ SavedConnection {+ enabled: false,+ credentials: Map::new(),+ },+ );+ write_connection_file(dir.path(), &conns).expect("write");++ let reloaded = read_connection_file(dir.path());+ assert_eq!(reloaded.len(), 1);+ assert!(reloaded.contains_key("slack"));+ assert!(!reloaded["slack"].enabled);+ // The temp file must be renamed away, not left behind.+ assert!(!dir.path().join("connections.tmp").exists());+ }}diff --git a/crates/screenpipe-core/src/pipes/preset_fallback.rs b/crates/screenpipe-core/src/pipes/preset_fallback.rsindex 85c3bb62d..120a30949 100644--- a/crates/screenpipe-core/src/pipes/preset_fallback.rs+++ b/crates/screenpipe-core/src/pipes/preset_fallback.rs@@ -200,14 +200,37 @@ impl PresetFallbackRegistry {}}+ /// Persist state to disk, logging (rather than swallowing) any failure.+ ///+ /// A silently-dropped write here means tripped breakers, cooldown timers and+ /// backoff counts never reach disk: on restart the registry reloads stale+ /// state and may re-pick a preset that is rate-limited or out of credits, or+ /// keep a recovered preset closed. We can't propagate from the call sites+ /// (they're fire-and-forget), but a `warn!` makes the failure observable+ /// instead of invisible — same lesson as the store.bin wipe fix.fn persist(&self, state: &PersistedState) {- // Atomic write: write to temp, rename+ if let Err(e) = self.try_persist(state) {+ warn!(+ "failed to persist preset fallback state to {:?}: {}",+ self.persist_path, e+ );+ }+ }++ /// Atomic write: serialize to a temp file, then rename over the target.+ /// Every failure mode (serialize, write, rename) is surfaced as an error+ /// rather than dropped. On a failed rename the temp file is removed so a+ /// stale `.json.tmp` can't linger next to the real state.+ fn try_persist(&self, state: &PersistedState) -> std::io::Result<()> {let tmp = self.persist_path.with_extension("json.tmp");- if let Ok(json) = serde_json::to_string_pretty(state) {- if std::fs::write(&tmp, &json).is_ok() {- let _ = std::fs::rename(&tmp, &self.persist_path);- }+ let json = serde_json::to_string_pretty(state)+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;+ std::fs::write(&tmp, &json)?;+ if let Err(e) = std::fs::rename(&tmp, &self.persist_path) {+ let _ = std::fs::remove_file(&tmp);+ return Err(e);}+ Ok(())}/// Pick the best preset from the list, respecting circuit breakers.@@ -593,4 +616,43 @@ mod tests {let registry = fresh_registry("empty");assert_eq!(registry.pick_preset_with_floor(&[], 0), None);}++ /// A tripped breaker must actually survive on disk: persist then reload via+ /// a fresh registry and confirm the state came back. Guards against a+ /// `persist` that silently writes nothing (the store.bin failure class).+ #[test]+ fn test_state_persists_across_reload() {+ let dir = std::env::temp_dir().join("sp_preset_fallback_reload");+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();++ {+ let registry = PresetFallbackRegistry::new(&dir);+ assert!(registry.record_failure("primary", Some("rate_limited")));+ }++ // A brand-new registry reads only what made it to disk.+ let reloaded = PresetFallbackRegistry::new(&dir);+ let state = reloaded.state.lock().unwrap_or_else(|e| e.into_inner());+ let breaker = state.presets.get("primary").expect("breaker persisted");+ assert_eq!(breaker.state, BreakerState::Open);+ assert_eq!(breaker.failure_count, 1);+ }++ /// A write that cannot land (parent directory missing) must surface an+ /// error from `try_persist` rather than being swallowed. Cross-platform:+ /// writing into a non-existent directory fails on every OS.+ #[test]+ fn test_persist_reports_error_when_unwritable() {+ let dir = std::env::temp_dir().join("sp_preset_fallback_missing_parent");+ let _ = std::fs::remove_dir_all(&dir); // ensure the parent does not exist+ let registry = PresetFallbackRegistry::new(&dir);++ let err = registry+ .try_persist(&PersistedState::default())+ .expect_err("persist into a missing directory must error");+ // And no stray temp file should be left behind.+ assert!(!dir.join("ai_preset_fallback.json.tmp").exists());+ drop(err);+ }}