Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(enterprise): prepare exports without waiting for payload reads (#7596)

Ezra Ellette12a93ab65fa0
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs b/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs
index e1c81ce45..e7bdf29ba 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs
@@ -279,6 +279,8 @@ mod tests {
async fn sync_retry_failure_reaches_support_after_collection_and_redaction() {
let dir = tempfile::tempdir().unwrap();
let report = crate::enterprise::sync::tests::check_sync_retry_failure(dir.path()).await;
+ assert!(report.contains("enterprise upload storage prepare export failed"));
+ assert!(report.contains("frame storage: payload read deadline exceeded"));
assert!(report.contains("frame storage: read revision changed"));
assert!(report.contains("pending work retained, retrying in 300s"));
assert!(!report.contains("backing off 3600s"));
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
index 5e86e424b..bd2f827a9 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
@@ -559,6 +559,11 @@ pub enum EnterpriseSyncError {
Network(String),
#[error("enterprise upload configuration blocked: {0}")]
Configuration(String),
+ #[error("enterprise upload storage {operation} failed: {cause}")]
+ Storage {
+ operation: &'static str,
+ cause: String,
+ },
/// The control plane's upload ticket pointed at a host outside the
/// MDM-pinned destination list (`SCREENPIPE_ENTERPRISE_UPLOAD_PINNED_HOSTS`).
/// This is the destination-pin tamper signal — permanent, loud, never
@@ -3788,9 +3793,19 @@ pub(crate) mod tests {
.unwrap();
assert_eq!(std::fs::read(&cursor_path).unwrap(), before);
reply
- .send(Err(EnterpriseSyncError::Configuration(
- "frame storage: read revision changed; retry the complete query".into(),
- )))
+ .send(Err(EnterpriseSyncError::Storage {
+ operation: if attempt % 2 == 0 {
+ "admit export"
+ } else {
+ "prepare export"
+ },
+ cause: if attempt % 2 == 0 {
+ "frame storage: read revision changed; retry the complete query"
+ } else {
+ "frame storage: payload read deadline exceeded"
+ }
+ .into(),
+ }))
.unwrap();
tokio::time::timeout(Duration::from_secs(10), async {
while std::fs::read_to_string(&log_path)
@@ -3867,6 +3882,8 @@ pub(crate) mod tests {
.find(|r| r.url.path() == "/support-upload")
.unwrap();
let logged = std::fs::read_to_string(log_path).unwrap();
+ assert!(logged.contains("enterprise upload storage prepare export failed"));
+ assert!(logged.contains("frame storage: payload read deadline exceeded"));
assert!(logged.contains("frame storage: read revision changed"));
assert!(logged.contains("pending work retained, retrying in 300s"));
assert!(!logged.contains("backing off 3600s"));
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
index d681313f1..0445cbae5 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
@@ -203,10 +203,13 @@ mod imp {
async fn begin(
db: Arc<screenpipe_db::DatabaseManager>,
) -> Result<Self, EnterpriseSyncError> {
- let token = db
- .storage_export_token()
- .await
- .map_err(|error| EnterpriseSyncError::Configuration(error.to_string()))?;
+ let token =
+ db.storage_export_token()
+ .await
+ .map_err(|error| EnterpriseSyncError::Storage {
+ operation: "prepare export",
+ cause: error.to_string(),
+ })?;
Ok(Self { db, token })
}
}
@@ -219,7 +222,10 @@ mod imp {
self.token
.admit(&self.db.pool)
.await
- .map_err(|error| EnterpriseSyncError::Configuration(error.to_string()))
+ .map_err(|error| EnterpriseSyncError::Storage {
+ operation: "admit export",
+ cause: error.to_string(),
+ })
}
}
@@ -2024,10 +2030,15 @@ mod imp {
let result =
sync::run_one_sync(&cfg, &mut cursor, &local, &reqwest::Client::new()).await;
if delete_selected {
- assert!(result
- .unwrap_err()
- .to_string()
- .contains("read revision changed"));
+ let error = result.unwrap_err();
+ assert!(matches!(
+ &error,
+ EnterpriseSyncError::Storage {
+ operation: "admit export",
+ ..
+ }
+ ));
+ assert!(error.to_string().contains("read revision changed"));
assert!(
!cfg.cursor_path.exists(),
"revoked data must not be checkpointed"
@@ -2054,6 +2065,17 @@ mod imp {
"upload admission must preserve the new recording"
);
db.close().await;
+ let error = StorageExport::begin(db)
+ .await
+ .err()
+ .expect("closed storage refused");
+ assert!(matches!(
+ error,
+ EnterpriseSyncError::Storage {
+ operation: "prepare export",
+ ..
+ }
+ ));
}
}
diff --git a/crates/screenpipe-db/src/storage/reader.rs b/crates/screenpipe-db/src/storage/reader.rs
index a6401ccc2..db13643c0 100644
--- a/crates/screenpipe-db/src/storage/reader.rs
+++ b/crates/screenpipe-db/src/storage/reader.rs
@@ -117,11 +117,29 @@ impl DatabaseManager {
.await
}
- /// Pin an export against deletion and privacy changes, while allowing
- /// new captures to continue. Each page still uses its own consistent read;
- /// ordinary inserts must not invalidate an already prepared upload.
+ /// Remember deletion/privacy revisions before assembling an export. Each
+ /// page owns its payload snapshot; the export itself only needs metadata.
+ /// Do not queue this check behind payload readers or pin their files for
+ /// the duration of a network upload. Final admission still rejects any
+ /// deletion/privacy change committed since this single-statement read.
pub async fn storage_export_token(&self) -> Result<StorageReadToken, sqlx::Error> {
- self.read_snapshot(self.storage_read_token()).await?
+ let Some(storage) = &self.storage else {
+ return self.storage_read_token().await;
+ };
+ if storage.closing.is_cancelled() {
+ return Err(sqlx::Error::PoolClosed);
+ }
+ let (revision, revocation) = sqlx::query_as(
+ "SELECT revision,(SELECT revision FROM _storage_revocation WHERE id=1) FROM storage_metadata WHERE singleton=1",
+ )
+ .fetch_one(&mut *self.acquire_read().await?)
+ .await?;
+ Ok(StorageReadToken {
+ revision,
+ revocation: Some(revocation),
+ storage: Some(Arc::clone(storage)),
+ _lease: None,
+ })
}
pub fn storage_mode(&self) -> super::StorageMode {
@@ -511,6 +529,88 @@ mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
+ #[tokio::test]
+ async fn export_metadata_does_not_wait_for_payload_read_lanes() {
+ let root = tempfile::tempdir().unwrap();
+ let db = Arc::new(
+ DatabaseManager::new_hybrid(
+ root.path(),
+ screenpipe_config::DbConfig {
+ read_pool_max: 2,
+ read_pool_min: 1,
+ ..Default::default()
+ },
+ Default::default(),
+ )
+ .await
+ .unwrap(),
+ );
+ let (started, ready) = tokio::sync::oneshot::channel();
+ let (release, released) = tokio::sync::oneshot::channel();
+ let reader = Arc::clone(&db);
+ let pending = tokio::spawn(async move {
+ reader
+ .read_snapshot(async {
+ started.send(()).unwrap();
+ released.await.unwrap();
+ })
+ .await
+ .unwrap();
+ });
+ ready.await.unwrap();
+ // The only payload lane is occupied. Its reserved pool connection is
+ // still available for metadata and final privacy admission.
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-12','capture while reading')").await.unwrap();
+ let export =
+ tokio::time::timeout(std::time::Duration::from_secs(1), db.storage_export_token())
+ .await;
+ release.send(()).unwrap();
+ pending.await.unwrap();
+ let token = export
+ .expect("metadata queued behind payload reads")
+ .unwrap();
+ drop(token.admit(&db.pool).await.unwrap());
+ let payloads = db.frame_payloads(&[1], Projection::All).await.unwrap();
+ assert_eq!(
+ payloads[&1].full_text.as_deref(),
+ Some("capture while reading")
+ );
+ drop(token);
+ db.close().await;
+ }
+
+ #[tokio::test]
+ async fn export_metadata_during_a_write_still_revokes_on_commit() {
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ db.execute_raw_sql_write(
+ "INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-12','private')",
+ )
+ .await
+ .unwrap();
+ let mut writer = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::query("DELETE FROM frames WHERE id=1")
+ .execute(&mut **writer.conn())
+ .await
+ .unwrap();
+ let export =
+ tokio::time::timeout(std::time::Duration::from_secs(1), db.storage_export_token())
+ .await;
+ writer.commit().await.unwrap();
+ let token = export
+ .expect("metadata waited for writer admission")
+ .unwrap();
+ assert!(is_revision_change(
+ &token.admit(&db.pool).await.unwrap_err()
+ ));
+ // Metadata has no payload files to pin for an entire network upload.
+ assert!(db.storage.as_ref().unwrap().leases.try_write().is_ok());
+ drop(token);
+ db.close().await;
+ }
+
#[tokio::test]
async fn export_admission_allows_recording_but_revokes_deleted_or_redacted_data() {
let root = tempfile::tempdir().unwrap();