Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(connections,pipes): prevent silent data loss on persistence paths (#4120)

Louis Beaumont1ef6f6cae04b
diff --git a/crates/screenpipe-core/src/connections/sync.rs b/crates/screenpipe-core/src/connections/sync.rs
index 722640b83..626fd8352 100644
--- a/crates/screenpipe-core/src/connections/sync.rs
+++ b/crates/screenpipe-core/src/connections/sync.rs
@@ -711,10 +711,7 @@ pub async fn apply_manifest_to_disk(
if let Err(e) = store.set_json(&secret_key, token).await {
errors.push(format!("failed to save oauth token {}: {}", key, e));
}
- } else if let Err(e) = std::fs::write(
- connection_oauth_file_path(screenpipe_dir, key),
- serde_json::to_string_pretty(token).unwrap_or_default(),
- ) {
+ } else if let Err(e) = write_oauth_token_file(screenpipe_dir, key, token) {
errors.push(format!("failed to write oauth token {}: {}", key, e));
}
@@ -809,10 +806,7 @@ pub async fn apply_manifest_to_disk(
errors.push(format!("missing oauth token for {}", key));
continue;
};
- if let Err(e) = std::fs::write(
- connection_oauth_file_path(screenpipe_dir, key),
- serde_json::to_string_pretty(token).unwrap_or_default(),
- ) {
+ if let Err(e) = write_oauth_token_file(screenpipe_dir, key, token) {
errors.push(format!("failed to write oauth token {}: {}", key, e));
}
connection_file.remove(key);
@@ -852,13 +846,39 @@ pub async fn apply_manifest_to_disk(
errors
}
+/// Atomic write of a per-connection OAuth token file (tmp + rename), matching
+/// write_connection_file. The previous inline `std::fs::write(..).
+/// unwrap_or_default()` was non-atomic (a crash mid-write truncates the token)
+/// and would silently persist an empty `""` token on a serialize failure; here
+/// the serialize error is surfaced and the old token survives a failed write.
+fn write_oauth_token_file(
+ screenpipe_dir: &Path,
+ manifest_key: &str,
+ token: &Value,
+) -> Result<(), String> {
+ let path = connection_oauth_file_path(screenpipe_dir, manifest_key);
+ let tmp = path.with_extension("tmp");
+ let json = serde_json::to_string_pretty(token).map_err(|e| format!("serialize: {e}"))?;
+ std::fs::write(&tmp, &json).map_err(|e| format!("write tmp {:?}: {}", tmp, e))?;
+ std::fs::rename(&tmp, &path).map_err(|e| format!("rename {:?}: {}", path, e))?;
+ Ok(())
+}
+
+/// Atomic write of connections.json (tmp file + rename), matching
+/// write_connection_tombstones. A direct write could truncate the file if it
+/// failed partway (crash, disk full), wiping every saved connection/credential;
+/// the temp-then-rename keeps the previous good file intact until the new one
+/// is fully written.
fn write_connection_file(
screenpipe_dir: &Path,
connections: &HashMap<String, SavedConnection>,
) -> Result<(), String> {
let path = screenpipe_dir.join("connections.json");
+ let tmp = path.with_extension("tmp");
let json = serde_json::to_string_pretty(connections).map_err(|e| format!("serialize: {e}"))?;
- std::fs::write(&path, &json).map_err(|e| format!("write {}: {}", path.display(), e))
+ std::fs::write(&tmp, &json).map_err(|e| format!("write tmp {:?}: {}", tmp, e))?;
+ std::fs::rename(&tmp, &path).map_err(|e| format!("rename {:?}: {}", path, e))?;
+ Ok(())
}
// ---------------------------------------------------------------------------
@@ -1164,4 +1184,94 @@ mod tests {
assert_eq!(manifest.connections["notion"].integration_id, "notion");
assert!(!manifest.connections["notion"].is_oauth);
}
+
+ #[test]
+ fn write_connection_file_preserves_existing_on_failed_write() {
+ // A failed write must NOT destroy the previously-saved connections.
+ // Force the temp write to fail by occupying the temp path with a
+ // directory; the atomic writer then errors and leaves connections.json
+ // untouched. A non-atomic direct write would clobber it instead — so
+ // this test fails against the old implementation and passes against the
+ // atomic one.
+ let dir = TempDir::new().unwrap();
+
+ let mut good = HashMap::new();
+ good.insert(
+ "gmail".to_string(),
+ SavedConnection {
+ enabled: true,
+ credentials: Map::new(),
+ },
+ );
+ write_connection_file(dir.path(), &good).expect("initial write");
+ assert!(read_connection_file(dir.path()).contains_key("gmail"));
+
+ // Block the temp path so the replacement write cannot land.
+ fs::create_dir(dir.path().join("connections.tmp")).unwrap();
+
+ let mut replacement = HashMap::new();
+ replacement.insert(
+ "outlook".to_string(),
+ SavedConnection {
+ enabled: true,
+ credentials: Map::new(),
+ },
+ );
+ let result = write_connection_file(dir.path(), &replacement);
+
+ assert!(result.is_err(), "write into a blocked temp path must error");
+ let reloaded = read_connection_file(dir.path());
+ assert!(
+ reloaded.contains_key("gmail"),
+ "existing connections must survive a failed write"
+ );
+ assert!(!reloaded.contains_key("outlook"));
+ }
+
+ #[test]
+ fn write_oauth_token_file_preserves_existing_on_failed_write() {
+ // A failed token write must not destroy the existing token. Block the
+ // temp path with a directory to force failure, then assert the previous
+ // token is intact and the call errors. Fails against the old inline
+ // direct write (which would overwrite); passes against the atomic one.
+ let dir = TempDir::new().unwrap();
+ let good = serde_json::json!({ "access_token": "good" });
+ write_oauth_token_file(dir.path(), "gmail", &good).expect("initial write");
+
+ let path = connection_oauth_file_path(dir.path(), "gmail");
+ assert!(path.exists());
+
+ fs::create_dir(path.with_extension("tmp")).unwrap();
+
+ let replacement = serde_json::json!({ "access_token": "new" });
+ let result = write_oauth_token_file(dir.path(), "gmail", &replacement);
+
+ assert!(result.is_err(), "write into a blocked temp path must error");
+ let reloaded: Value = serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap();
+ assert_eq!(
+ reloaded["access_token"], "good",
+ "existing token must survive a failed write"
+ );
+ }
+
+ #[test]
+ fn write_connection_file_roundtrips_and_leaves_no_temp() {
+ let dir = TempDir::new().unwrap();
+ let mut conns = HashMap::new();
+ conns.insert(
+ "slack".to_string(),
+ SavedConnection {
+ enabled: false,
+ credentials: Map::new(),
+ },
+ );
+ write_connection_file(dir.path(), &conns).expect("write");
+
+ let reloaded = read_connection_file(dir.path());
+ assert_eq!(reloaded.len(), 1);
+ assert!(reloaded.contains_key("slack"));
+ assert!(!reloaded["slack"].enabled);
+ // The temp file must be renamed away, not left behind.
+ assert!(!dir.path().join("connections.tmp").exists());
+ }
}
diff --git a/crates/screenpipe-core/src/pipes/preset_fallback.rs b/crates/screenpipe-core/src/pipes/preset_fallback.rs
index 85c3bb62d..120a30949 100644
--- a/crates/screenpipe-core/src/pipes/preset_fallback.rs
+++ b/crates/screenpipe-core/src/pipes/preset_fallback.rs
@@ -200,14 +200,37 @@ impl PresetFallbackRegistry {
}
}
+ /// Persist state to disk, logging (rather than swallowing) any failure.
+ ///
+ /// A silently-dropped write here means tripped breakers, cooldown timers and
+ /// backoff counts never reach disk: on restart the registry reloads stale
+ /// state and may re-pick a preset that is rate-limited or out of credits, or
+ /// keep a recovered preset closed. We can't propagate from the call sites
+ /// (they're fire-and-forget), but a `warn!` makes the failure observable
+ /// instead of invisible — same lesson as the store.bin wipe fix.
fn persist(&self, state: &PersistedState) {
- // Atomic write: write to temp, rename
+ if let Err(e) = self.try_persist(state) {
+ warn!(
+ "failed to persist preset fallback state to {:?}: {}",
+ self.persist_path, e
+ );
+ }
+ }
+
+ /// Atomic write: serialize to a temp file, then rename over the target.
+ /// Every failure mode (serialize, write, rename) is surfaced as an error
+ /// rather than dropped. On a failed rename the temp file is removed so a
+ /// stale `.json.tmp` can't linger next to the real state.
+ fn try_persist(&self, state: &PersistedState) -> std::io::Result<()> {
let tmp = self.persist_path.with_extension("json.tmp");
- if let Ok(json) = serde_json::to_string_pretty(state) {
- if std::fs::write(&tmp, &json).is_ok() {
- let _ = std::fs::rename(&tmp, &self.persist_path);
- }
+ let json = serde_json::to_string_pretty(state)
+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
+ std::fs::write(&tmp, &json)?;
+ if let Err(e) = std::fs::rename(&tmp, &self.persist_path) {
+ let _ = std::fs::remove_file(&tmp);
+ return Err(e);
}
+ Ok(())
}
/// Pick the best preset from the list, respecting circuit breakers.
@@ -593,4 +616,43 @@ mod tests {
let registry = fresh_registry("empty");
assert_eq!(registry.pick_preset_with_floor(&[], 0), None);
}
+
+ /// A tripped breaker must actually survive on disk: persist then reload via
+ /// a fresh registry and confirm the state came back. Guards against a
+ /// `persist` that silently writes nothing (the store.bin failure class).
+ #[test]
+ fn test_state_persists_across_reload() {
+ let dir = std::env::temp_dir().join("sp_preset_fallback_reload");
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+
+ {
+ let registry = PresetFallbackRegistry::new(&dir);
+ assert!(registry.record_failure("primary", Some("rate_limited")));
+ }
+
+ // A brand-new registry reads only what made it to disk.
+ let reloaded = PresetFallbackRegistry::new(&dir);
+ let state = reloaded.state.lock().unwrap_or_else(|e| e.into_inner());
+ let breaker = state.presets.get("primary").expect("breaker persisted");
+ assert_eq!(breaker.state, BreakerState::Open);
+ assert_eq!(breaker.failure_count, 1);
+ }
+
+ /// A write that cannot land (parent directory missing) must surface an
+ /// error from `try_persist` rather than being swallowed. Cross-platform:
+ /// writing into a non-existent directory fails on every OS.
+ #[test]
+ fn test_persist_reports_error_when_unwritable() {
+ let dir = std::env::temp_dir().join("sp_preset_fallback_missing_parent");
+ let _ = std::fs::remove_dir_all(&dir); // ensure the parent does not exist
+ let registry = PresetFallbackRegistry::new(&dir);
+
+ let err = registry
+ .try_persist(&PersistedState::default())
+ .expect_err("persist into a missing directory must error");
+ // And no stray temp file should be left behind.
+ assert!(!dir.join("ai_preset_fallback.json.tmp").exists());
+ drop(err);
+ }
}