YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(enterprise): prepare exports without waiting for payload reads (#7596)
Ezra Ellette12a93ab65fa0
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs b/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rsindex e1c81ce45..e7bdf29ba 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs@@ -279,6 +279,8 @@ mod tests {async fn sync_retry_failure_reaches_support_after_collection_and_redaction() {let dir = tempfile::tempdir().unwrap();let report = crate::enterprise::sync::tests::check_sync_retry_failure(dir.path()).await;+ assert!(report.contains("enterprise upload storage prepare export failed"));+ assert!(report.contains("frame storage: payload read deadline exceeded"));assert!(report.contains("frame storage: read revision changed"));assert!(report.contains("pending work retained, retrying in 300s"));assert!(!report.contains("backing off 3600s"));diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rsindex 5e86e424b..bd2f827a9 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs@@ -559,6 +559,11 @@ pub enum EnterpriseSyncError {Network(String),#[error("enterprise upload configuration blocked: {0}")]Configuration(String),+ #[error("enterprise upload storage {operation} failed: {cause}")]+ Storage {+ operation: &'static str,+ cause: String,+ },/// The control plane's upload ticket pointed at a host outside the/// MDM-pinned destination list (`SCREENPIPE_ENTERPRISE_UPLOAD_PINNED_HOSTS`)./// This is the destination-pin tamper signal — permanent, loud, never@@ -3788,9 +3793,19 @@ pub(crate) mod tests {.unwrap();assert_eq!(std::fs::read(&cursor_path).unwrap(), before);reply- .send(Err(EnterpriseSyncError::Configuration(- "frame storage: read revision changed; retry the complete query".into(),- )))+ .send(Err(EnterpriseSyncError::Storage {+ operation: if attempt % 2 == 0 {+ "admit export"+ } else {+ "prepare export"+ },+ cause: if attempt % 2 == 0 {+ "frame storage: read revision changed; retry the complete query"+ } else {+ "frame storage: payload read deadline exceeded"+ }+ .into(),+ })).unwrap();tokio::time::timeout(Duration::from_secs(10), async {while std::fs::read_to_string(&log_path)@@ -3867,6 +3882,8 @@ pub(crate) mod tests {.find(|r| r.url.path() == "/support-upload").unwrap();let logged = std::fs::read_to_string(log_path).unwrap();+ assert!(logged.contains("enterprise upload storage prepare export failed"));+ assert!(logged.contains("frame storage: payload read deadline exceeded"));assert!(logged.contains("frame storage: read revision changed"));assert!(logged.contains("pending work retained, retrying in 300s"));assert!(!logged.contains("backing off 3600s"));diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rsindex d681313f1..0445cbae5 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs@@ -203,10 +203,13 @@ mod imp {async fn begin(db: Arc<screenpipe_db::DatabaseManager>,) -> Result<Self, EnterpriseSyncError> {- let token = db- .storage_export_token()- .await- .map_err(|error| EnterpriseSyncError::Configuration(error.to_string()))?;+ let token =+ db.storage_export_token()+ .await+ .map_err(|error| EnterpriseSyncError::Storage {+ operation: "prepare export",+ cause: error.to_string(),+ })?;Ok(Self { db, token })}}@@ -219,7 +222,10 @@ mod imp {self.token.admit(&self.db.pool).await- .map_err(|error| EnterpriseSyncError::Configuration(error.to_string()))+ .map_err(|error| EnterpriseSyncError::Storage {+ operation: "admit export",+ cause: error.to_string(),+ })}}@@ -2024,10 +2030,15 @@ mod imp {let result =sync::run_one_sync(&cfg, &mut cursor, &local, &reqwest::Client::new()).await;if delete_selected {- assert!(result- .unwrap_err()- .to_string()- .contains("read revision changed"));+ let error = result.unwrap_err();+ assert!(matches!(+ &error,+ EnterpriseSyncError::Storage {+ operation: "admit export",+ ..+ }+ ));+ assert!(error.to_string().contains("read revision changed"));assert!(!cfg.cursor_path.exists(),"revoked data must not be checkpointed"@@ -2054,6 +2065,17 @@ mod imp {"upload admission must preserve the new recording");db.close().await;+ let error = StorageExport::begin(db)+ .await+ .err()+ .expect("closed storage refused");+ assert!(matches!(+ error,+ EnterpriseSyncError::Storage {+ operation: "prepare export",+ ..+ }+ ));}}diff --git a/crates/screenpipe-db/src/storage/reader.rs b/crates/screenpipe-db/src/storage/reader.rsindex a6401ccc2..db13643c0 100644--- a/crates/screenpipe-db/src/storage/reader.rs+++ b/crates/screenpipe-db/src/storage/reader.rs@@ -117,11 +117,29 @@ impl DatabaseManager {.await}- /// Pin an export against deletion and privacy changes, while allowing- /// new captures to continue. Each page still uses its own consistent read;- /// ordinary inserts must not invalidate an already prepared upload.+ /// Remember deletion/privacy revisions before assembling an export. Each+ /// page owns its payload snapshot; the export itself only needs metadata.+ /// Do not queue this check behind payload readers or pin their files for+ /// the duration of a network upload. Final admission still rejects any+ /// deletion/privacy change committed since this single-statement read.pub async fn storage_export_token(&self) -> Result<StorageReadToken, sqlx::Error> {- self.read_snapshot(self.storage_read_token()).await?+ let Some(storage) = &self.storage else {+ return self.storage_read_token().await;+ };+ if storage.closing.is_cancelled() {+ return Err(sqlx::Error::PoolClosed);+ }+ let (revision, revocation) = sqlx::query_as(+ "SELECT revision,(SELECT revision FROM _storage_revocation WHERE id=1) FROM storage_metadata WHERE singleton=1",+ )+ .fetch_one(&mut *self.acquire_read().await?)+ .await?;+ Ok(StorageReadToken {+ revision,+ revocation: Some(revocation),+ storage: Some(Arc::clone(storage)),+ _lease: None,+ })}pub fn storage_mode(&self) -> super::StorageMode {@@ -511,6 +529,88 @@ mod tests {use super::*;use std::sync::atomic::{AtomicUsize, Ordering};+ #[tokio::test]+ async fn export_metadata_does_not_wait_for_payload_read_lanes() {+ let root = tempfile::tempdir().unwrap();+ let db = Arc::new(+ DatabaseManager::new_hybrid(+ root.path(),+ screenpipe_config::DbConfig {+ read_pool_max: 2,+ read_pool_min: 1,+ ..Default::default()+ },+ Default::default(),+ )+ .await+ .unwrap(),+ );+ let (started, ready) = tokio::sync::oneshot::channel();+ let (release, released) = tokio::sync::oneshot::channel();+ let reader = Arc::clone(&db);+ let pending = tokio::spawn(async move {+ reader+ .read_snapshot(async {+ started.send(()).unwrap();+ released.await.unwrap();+ })+ .await+ .unwrap();+ });+ ready.await.unwrap();+ // The only payload lane is occupied. Its reserved pool connection is+ // still available for metadata and final privacy admission.+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-12','capture while reading')").await.unwrap();+ let export =+ tokio::time::timeout(std::time::Duration::from_secs(1), db.storage_export_token())+ .await;+ release.send(()).unwrap();+ pending.await.unwrap();+ let token = export+ .expect("metadata queued behind payload reads")+ .unwrap();+ drop(token.admit(&db.pool).await.unwrap());+ let payloads = db.frame_payloads(&[1], Projection::All).await.unwrap();+ assert_eq!(+ payloads[&1].full_text.as_deref(),+ Some("capture while reading")+ );+ drop(token);+ db.close().await;+ }++ #[tokio::test]+ async fn export_metadata_during_a_write_still_revokes_on_commit() {+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ db.execute_raw_sql_write(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-12','private')",+ )+ .await+ .unwrap();+ let mut writer = db.begin_immediate_with_retry().await.unwrap();+ sqlx::query("DELETE FROM frames WHERE id=1")+ .execute(&mut **writer.conn())+ .await+ .unwrap();+ let export =+ tokio::time::timeout(std::time::Duration::from_secs(1), db.storage_export_token())+ .await;+ writer.commit().await.unwrap();+ let token = export+ .expect("metadata waited for writer admission")+ .unwrap();+ assert!(is_revision_change(+ &token.admit(&db.pool).await.unwrap_err()+ ));+ // Metadata has no payload files to pin for an entire network upload.+ assert!(db.storage.as_ref().unwrap().leases.try_write().is_ok());+ drop(token);+ db.close().await;+ }+#[tokio::test]async fn export_admission_allows_recording_but_revokes_deleted_or_redacted_data() {let root = tempfile::tempdir().unwrap();