Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(enterprise): filter changed records without failing uploads (#7667)

Ezra Ellette8001273725a1
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs b/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs
index 1bdf13ed4..2e08c677d 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs
@@ -302,8 +302,9 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
let report = crate::enterprise::sync::tests::check_sync_retry_failure(dir.path()).await;
assert!(report.contains("enterprise upload storage prepare export failed"));
+ assert!(report.contains("enterprise upload storage filter export failed"));
assert!(report.contains("frame storage: payload read deadline exceeded"));
- assert!(report.contains("frame storage: read revision changed"));
+ assert!(report.contains("database disk image is malformed"));
assert!(report.contains("pending work retained, retrying in 300s"));
assert!(!report.contains("backing off 3600s"));
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
index efcbccc68..23e45e1f9 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
@@ -1220,7 +1220,7 @@ pub(super) async fn fulfill_requests(
.await?;
// JPEG pages are heavier than telemetry pages. Yield to live
// sync after one acknowledged image page, retaining its cursor.
- if complete || cursor.boundary.backfill_images.is_some() {
+ if complete || page.pending_read || cursor.boundary.backfill_images.is_some() {
break;
}
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/export_filter.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/export_filter.rs
new file mode 100644
index 000000000..3d3dc6894
--- /dev/null
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/export_filter.rs
@@ -0,0 +1,515 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+use super::*;
+
+/// Absolute offsets recomputed from current source rows, including previously
+/// acknowledged rows at the same timestamp. Deleting an earlier row shifts
+/// those offsets too; subtracting only deleted outgoing rows would skip data.
+#[derive(Default)]
+pub struct ExportFilterReport {
+ pub frames: Option<(String, u32)>,
+ pub audio: Option<(String, u32)>,
+ pub ui: Option<(String, u32)>,
+ pub memories: Option<(String, u32)>,
+ pub parsed: Option<(String, u32)>,
+}
+
+impl ExportFilterReport {
+ pub(super) fn apply(self, cursor: &mut Cursor) {
+ for (current, offset, fresh) in [
+ (
+ &cursor.last_frame_ts,
+ &mut cursor.boundary.frames,
+ self.frames,
+ ),
+ (
+ &cursor.last_audio_ts,
+ &mut cursor.boundary.audio,
+ self.audio,
+ ),
+ (&cursor.last_ui_ts, &mut cursor.boundary.ui, self.ui),
+ (
+ &cursor.last_memory_ts,
+ &mut cursor.boundary.memories,
+ self.memories,
+ ),
+ (
+ &cursor.last_parsed_ts,
+ &mut cursor.boundary.parsed,
+ self.parsed,
+ ),
+ ] {
+ if let Some((timestamp, count)) = fresh {
+ if current.as_deref() == Some(timestamp.as_str()) {
+ *offset = count;
+ }
+ }
+ }
+ }
+}
+
+/// Each HTTP request is filtered again after any preceding network wait.
+/// A deletion while an earlier request is in flight must not leak through the
+/// already serialized remainder of a large page.
+pub(super) async fn filter_request(
+ cfg: &EnterpriseSyncConfig,
+ export: Option<&dyn ExportFilter>,
+ local: &dyn LocalApiClient,
+ body: Vec<u8>,
+ cursor: &mut Cursor,
+ ratings_only: bool,
+) -> Result<(Vec<u8>, SyncTickReport), EnterpriseSyncError> {
+ let mut frames = Vec::new();
+ let mut audio = Vec::new();
+ let mut ui = Vec::new();
+ let mut snapshots = Vec::new();
+ let mut memories = Vec::new();
+ let mut parsed = Vec::new();
+ let mut activities = Vec::new();
+ let mut feedback = Vec::new();
+ for line in body
+ .split(|byte| *byte == b'\n')
+ .filter(|line| !line.is_empty())
+ {
+ let record: TelemetryRecord = serde_json::from_slice(line)
+ .map_err(|e| EnterpriseSyncError::LocalApi(format!("decode prepared export: {e}")))?;
+ match record {
+ TelemetryRecord::Frame { frame, .. } => frames.push(frame),
+ TelemetryRecord::Audio { audio: row, .. } => audio.push(row),
+ TelemetryRecord::Ui { ui: row, .. } => ui.push(row),
+ TelemetryRecord::Snapshot { snapshot, .. } => snapshots.push(snapshot),
+ TelemetryRecord::Memory { memory, .. } => memories.push(memory),
+ TelemetryRecord::Parsed { parsed: row, .. } => parsed.push(row),
+ TelemetryRecord::Activity { activity, .. } => activities.push(activity),
+ TelemetryRecord::Feedback { feedback: row, .. } => feedback.push(row),
+ }
+ }
+ drop(body);
+ if let Some(export) = export {
+ export
+ .filter(
+ &mut ExportRows {
+ frames: &mut frames,
+ audio: &mut audio,
+ ui: &mut ui,
+ snapshots: &mut snapshots,
+ memories: &mut memories,
+ parsed: &mut parsed,
+ activities: &mut activities,
+ feedback: &mut feedback,
+ },
+ local,
+ )
+ .await?
+ .apply(cursor);
+ }
+ if ratings_only {
+ for row in &mut feedback {
+ row.comment = None;
+ row.snapshot = None;
+ row.context = serde_json::json!({});
+ }
+ }
+ let mut body = screenpipe_telemetry_wire::build_jsonl_with_semantic_streams(
+ &cfg.device_id,
+ &cfg.device_label,
+ &frames,
+ &parsed,
+ &activities,
+ &audio,
+ &ui,
+ &snapshots,
+ &memories,
+ );
+ body.extend_from_slice(&screenpipe_telemetry_wire::build_feedback_jsonl(
+ &cfg.device_id,
+ &cfg.device_label,
+ &feedback,
+ ));
+ let report = SyncTickReport {
+ frames: frames.len(),
+ parsed: parsed.len(),
+ activities: activities.len(),
+ audio: audio.len(),
+ ui: ui.len(),
+ snapshots: snapshots.len(),
+ memories: memories.len(),
+ feedback: feedback.len(),
+ bytes: body.len(),
+ ..Default::default()
+ };
+ Ok((body, report))
+}
+
+/// Borrow the outgoing page so filtering keeps the original selection and order.
+pub struct ExportRows<'a> {
+ pub frames: &'a mut Vec<FrameRow>,
+ pub audio: &'a mut Vec<AudioRow>,
+ pub ui: &'a mut Vec<UiEventRow>,
+ pub snapshots: &'a mut Vec<SnapshotRow>,
+ pub memories: &'a mut Vec<MemoryRow>,
+ pub parsed: &'a mut Vec<ParsedRow>,
+ pub activities: &'a mut Vec<ActivityRow>,
+ pub feedback: &'a mut Vec<FeedbackRow>,
+}
+
+impl ExportRows<'_> {
+ pub async fn refresh(
+ &mut self,
+ local: &dyn LocalApiClient,
+ ) -> Result<ExportFilterReport, EnterpriseSyncError> {
+ let mut report = ExportFilterReport::default();
+ report.frames = filter_current_rows(
+ self.frames,
+ FRAME_PAGE_LIMIT,
+ |r| r.frame_id,
+ |r| r.timestamp.as_str(),
+ |since, offset, limit| async move {
+ local.fetch_frames_since(Some(&since), offset, limit).await
+ },
+ )
+ .await?;
+ // The local audio API identifies ordinary audio by chunk + timestamp,
+ // and live meeting segments by a negative segment ID.
+ report.audio = filter_current_rows(
+ self.audio,
+ PAGE_LIMIT,
+ |r| (r.transcription_id, r.timestamp.clone()),
+ |r| r.timestamp.as_str(),
+ |since, offset, limit| async move {
+ local.fetch_audio_since(Some(&since), offset, limit).await
+ },
+ )
+ .await?;
+ report.ui = filter_current_rows(
+ self.ui,
+ PAGE_LIMIT,
+ |r| r.event_id,
+ |r| r.timestamp.as_str(),
+ |since, offset, limit| async move {
+ local
+ .fetch_ui_events_since(Some(&since), offset, limit)
+ .await
+ },
+ )
+ .await?;
+ report.memories = filter_current_rows(
+ self.memories,
+ PAGE_LIMIT,
+ |r| r.memory_id,
+ |r| r.created_at.as_str(),
+ |since, offset, limit| async move {
+ local
+ .fetch_memories_since(Some(&since), offset, limit)
+ .await
+ },
+ )
+ .await?;
+ report.parsed = filter_current_rows(
+ self.parsed,
+ PAGE_LIMIT,
+ |r| r.frame_id,
+ |r| r.timestamp.as_str(),
+ |since, offset, limit| async move {
+ local.fetch_parsed_since(Some(&since), offset, limit).await
+ },
+ )
+ .await?;
+ if let Some(first) = self.feedback.first() {
+ let mut since = first.updated_at.clone();
+ let wanted: std::collections::HashSet<_> = self
+ .feedback
+ .iter()
+ .map(|r| r.feedback_id.clone())
+ .collect();
+ let mut current = std::collections::HashMap::new();
+ let mut after = None;
+ loop {
+ let page = local
+ .fetch_feedback_since(Some(&since), after.as_deref(), PAGE_LIMIT)
+ .await?;
+ let more = page.len() == PAGE_LIMIT as usize;
+ let Some(last) = page.last() else {
+ break;
+ };
+ since = last.updated_at.clone();
+ after = Some(last.feedback_id.clone());
+ for row in page {
+ if wanted.contains(&row.feedback_id) {
+ current.insert(row.feedback_id.clone(), row);
+ }
+ }
+ if !more || current.len() == wanted.len() {
+ break;
+ }
+ }
+ self.feedback
+ .retain_mut(|row| match current.remove(&row.feedback_id) {
+ Some(fresh) => {
+ *row = fresh;
+ true
+ }
+ None => false,
+ });
+ }
+ if let Some(first) = self.activities.first() {
+ let current = local.fetch_activities_since(Some(&first.start_at)).await?;
+ let mut current: std::collections::HashMap<_, _> = current
+ .into_iter()
+ .map(|r| (r.activity_id.clone(), r))
+ .collect();
+ self.activities
+ .retain_mut(|row| match current.remove(&row.activity_id) {
+ Some(fresh) => {
+ *row = fresh;
+ true
+ }
+ None => false,
+ });
+ }
+ let mut snapshots = Vec::with_capacity(self.snapshots.len());
+ for row in self.snapshots.iter() {
+ let jpeg = match local.fetch_frame_jpeg(row.frame_id).await {
+ Ok(jpeg) => jpeg,
+ Err(EnterpriseSyncError::BackfillImages("screenshot_policy_disabled")) => None,
+ Err(error) => return Err(error),
+ };
+ if let Some(jpeg) = jpeg {
+ let id = row.frame_id;
+ let timestamp = row.timestamp.clone();
+ if let Some(fresh) =
+ tokio::task::spawn_blocking(move || snapshot_from_jpeg(id, timestamp, &jpeg))
+ .await
+ .map_err(|e| {
+ EnterpriseSyncError::LocalApi(format!("refresh snapshot: {e}"))
+ })?
+ {
+ snapshots.push(fresh);
+ }
+ }
+ }
+ *self.snapshots = snapshots;
+ Ok(report)
+ }
+}
+
+/// Re-read only the selected time range, page through timestamp ties, and keep
+/// the selected IDs that still exist. Current contents replace stale/redacted
+/// copies. New captures are left for the next page instead of joining this one.
+async fn filter_current_rows<T, K, F, Fut>(
+ selected: &mut Vec<T>,
+ limit: u32,
+ key: impl Fn(&T) -> K,
+ timestamp: impl Fn(&T) -> &str,
+ mut fetch: F,
+) -> Result<Option<(String, u32)>, EnterpriseSyncError>
+where
+ K: Eq + std::hash::Hash,
+ F: FnMut(String, u32, u32) -> Fut,
+ Fut: std::future::Future<Output = Result<Vec<T>, EnterpriseSyncError>>,
+{
+ let (Some(first), Some(last)) = (selected.first(), selected.last()) else {
+ return Ok(None);
+ };
+ let mut boundary_offset = 0u32;
+ let mut rows_at_boundary = 0u32;
+ let mut since = Some(timestamp(first).to_owned());
+ let last_ts = timestamp(last).to_owned();
+ let mut wanted = std::collections::HashMap::new();
+ for row in selected.iter() {
+ *wanted.entry(key(row)).or_insert(0usize) += 1;
+ }
+ let selected_count = selected.len();
+ let mut matched = 0usize;
+ let mut current: std::collections::HashMap<K, std::collections::VecDeque<T>> =
+ std::collections::HashMap::new();
+ let mut offset = 0;
+ loop {
+ let page = fetch(since.clone().unwrap(), offset, limit).await?;
+ let more = page.len() == limit as usize;
+ if page.is_empty() {
+ break;
+ }
+ advance_timestamp_boundary(&mut since, &mut offset, &page, &timestamp);
+ for row in page {
+ let at_boundary = timestamp(&row) == last_ts;
+ if at_boundary {
+ rows_at_boundary += 1;
+ }
+ let id = key(&row);
+ if let Some(count) = wanted.get(&id) {
+ let values = current.entry(id).or_default();
+ if values.len() < *count {
+ values.push_back(row);
+ matched += 1;
+ if at_boundary {
+ boundary_offset = rows_at_boundary;
+ }
+ }
+ }
+ }
+ if !more || since.as_deref().unwrap() > last_ts.as_str() || matched == selected_count {
+ break;
+ }
+ }
+ selected.retain_mut(
+ |row| match current.get_mut(&key(row)).and_then(|rows| rows.pop_front()) {
+ Some(fresh) => {
+ *row = fresh;
+ true
+ }
+ None => false,
+ },
+ );
+ // With no surviving selected row at this timestamp, replay from its start.
+ // Upload keys are idempotent; replay is preferable to skipping a new row.
+ Ok(Some((last_ts, boundary_offset)))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::{Arc, Mutex};
+
+ fn frame(id: i64, text: &str) -> FrameRow {
+ FrameRow {
+ frame_id: id,
+ timestamp: "2026-10-09T00:00:00Z".into(),
+ app_name: None,
+ window_name: None,
+ browser_url: None,
+ text: Some(text.into()),
+ accessibility_tree: None,
+ }
+ }
+
+ #[tokio::test]
+ async fn deletion_filters_only_selected_ids_and_recomputes_shifted_timestamp_offset() {
+ let mut selected = vec![
+ frame(2, "deleted"),
+ frame(3, "private"),
+ frame(4, "deleted tail"),
+ ];
+ // Row 1 was acknowledged earlier and has also been deleted. Row 0 is
+ // still present before this selection; row 5 is an unselected capture.
+ let current = Arc::new(vec![
+ frame(0, "earlier"),
+ frame(3, "[REDACTED]"),
+ frame(5, "new"),
+ ]);
+ let calls = Arc::new(Mutex::new(Vec::new()));
+ let offset = filter_current_rows(&mut selected, 2, |r| r.frame_id, |r| &r.timestamp, {
+ let calls = calls.clone();
+ move |since, offset, limit| {
+ let current = current.clone();
+ let calls = calls.clone();
+ async move {
+ calls.lock().unwrap().push((since, offset));
+ Ok(current
+ .iter()
+ .skip(offset as usize)
+ .take(limit as usize)
+ .cloned()
+ .collect())
+ }
+ }
+ })
+ .await
+ .unwrap();
+ assert_eq!(selected, vec![frame(3, "[REDACTED]")]);
+ assert_eq!(
+ offset,
+ Some(("2026-10-09T00:00:00Z".into(), 2)),
+ "count surviving earlier rows as well as selected rows"
+ );
+ assert_eq!(
+ calls
+ .lock()
+ .unwrap()
+ .iter()
+ .map(|(_, offset)| *offset)
+ .collect::<Vec<_>>(),
+ vec![0, 2]
+ );
+ }
+
+ #[tokio::test]
+ async fn completely_deleted_selection_keeps_the_next_capture_for_the_next_page() {
+ let mut selected = vec![frame(1, "deleted"), frame(2, "deleted")];
+ let offset = filter_current_rows(
+ &mut selected,
+ 2,
+ |r| r.frame_id,
+ |r| &r.timestamp,
+ |_, _, _| async { Ok(vec![frame(3, "new capture")]) },
+ )
+ .await
+ .unwrap();
+ assert!(selected.is_empty());
+ assert_eq!(offset, Some(("2026-10-09T00:00:00Z".into(), 0)));
+ }
+
+ #[tokio::test]
+ async fn source_order_and_repeated_keys_are_preserved() {
+ let mut selected = vec![frame(-2, "a"), frame(-2, "b"), frame(-3, "c")];
+ let current = Arc::new(vec![
+ frame(-1, "earlier"),
+ frame(-2, "a redacted"),
+ frame(-2, "b redacted"),
+ frame(-3, "c redacted"),
+ frame(-4, "new"),
+ ]);
+ let offset = filter_current_rows(
+ &mut selected,
+ 2,
+ |r| r.frame_id,
+ |r| &r.timestamp,
+ move |_, offset, limit| {
+ let current = current.clone();
+ async move {
+ Ok(current
+ .iter()
+ .skip(offset as usize)
+ .take(limit as usize)
+ .cloned()
+ .collect())
+ }
+ },
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ selected,
+ vec![
+ frame(-2, "a redacted"),
+ frame(-2, "b redacted"),
+ frame(-3, "c redacted")
+ ]
+ );
+ assert_eq!(offset, Some(("2026-10-09T00:00:00Z".into(), 4)));
+ }
+
+ #[tokio::test]
+ async fn genuine_read_failure_preserves_the_selection_and_original_cause() {
+ let mut selected = vec![frame(1, "pending")];
+ let before = selected.clone();
+ let error = filter_current_rows(
+ &mut selected,
+ 2,
+ |r| r.frame_id,
+ |r| &r.timestamp,
+ |_, _, _| async {
+ Err(EnterpriseSyncError::LocalApi(
+ "database disk image is malformed".into(),
+ ))
+ },
+ )
+ .await
+ .unwrap_err();
+ assert!(error
+ .to_string()
+ .contains("database disk image is malformed"));
+ assert_eq!(selected, before);
+ }
+}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs
index 562b81492..5e4738067 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs
@@ -39,12 +39,9 @@ pub(crate) async fn hydrate_frame_trees(
.await
.map_err(|e| EnterpriseSyncError::LocalApi(format!("prepare frame AX payloads: {e}")))??;
for frame in frames {
- frame.accessibility_tree = trees.remove(&frame.frame_id).ok_or_else(|| {
- EnterpriseSyncError::LocalApi(format!(
- "frame {} disappeared during AX export",
- frame.frame_id
- ))
- })?;
+ // Keep the selected ID and scan boundary even if retention removed its
+ // payload. The final export filter drops it before serialization.
+ frame.accessibility_tree = trees.remove(&frame.frame_id).flatten();
}
Ok(())
}
@@ -98,11 +95,15 @@ mod tests {
}
#[tokio::test]
- async fn retained_tree_reaches_wire_and_missing_frames_fail() {
+ async fn retained_tree_reaches_wire_and_missing_frames_are_left_for_filtering() {
for archived in [false, true] {
let dir = tempfile::tempdir().unwrap();
let db = if archived {
- DatabaseManager::new_hybrid(dir.path(), Default::default(), Default::default())
+ // This tiny synthetic fixture tests export contents, not disk
+ // admission. Match the DB suite's fixture configuration.
+ let mut options = screenpipe_db::storage::MigrationOptions::default();
+ options.budget.disk_reserve_bytes = 0;
+ DatabaseManager::new_hybrid(dir.path(), Default::default(), options)
.await
.unwrap()
} else {
@@ -144,9 +145,9 @@ mod tests {
"search"
);
assert!(frames[1].accessibility_tree.is_none());
- assert!(hydrate_frame_trees(&db, &mut [row(3)], false)
- .await
- .is_err());
+ let mut missing = vec![row(3)];
+ hydrate_frame_trees(&db, &mut missing, false).await.unwrap();
+ assert!(missing[0].accessibility_tree.is_none());
db.close().await;
}
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
index c352d654f..e89104ca2 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
@@ -19,7 +19,7 @@
//!
//! # Edge cases handled
//!
-//! - **Empty batch** — skip POST, advance no cursor, retry next tick
+//! - **Empty batch** — skip POST; confirmed deletions still advance the scan boundary
//! - **Network failure** — retain the cursor and retry on the normal sync cycle
//! - **4xx auth failure** — use the signed-in employee account to fetch the
//! rotated device key, then retry without advancing the cursor
@@ -313,13 +313,23 @@ impl Cursor {
/// inject a mock without spinning up the real server. Implemented in the
/// desktop crate against `LocalApiContext`.
#[async_trait::async_trait]
-pub trait ExportAdmission: Send + Sync {
- async fn admit(&self) -> Result<Option<tokio::sync::OwnedMutexGuard<()>>, EnterpriseSyncError>;
+pub trait ExportFilter: Send + Sync {
+ /// Filter the selected records against their current source state before
+ /// serialization. A deletion removes that record, never the whole batch.
+ async fn filter(
+ &self,
+ rows: &mut ExportRows<'_>,
+ local: &dyn LocalApiClient,
+ ) -> Result<ExportFilterReport, EnterpriseSyncError>;
}
+#[path = "export_filter.rs"]
+mod export_filter;
+pub use export_filter::{ExportFilterReport, ExportRows};
+
#[async_trait::async_trait]
pub trait LocalApiClient: Send + Sync {
- async fn begin_export(&self) -> Result<Option<Box<dyn ExportAdmission>>, EnterpriseSyncError> {
+ async fn begin_export(&self) -> Result<Option<Box<dyn ExportFilter>>, EnterpriseSyncError> {
Ok(None)
}
@@ -576,6 +586,28 @@ pub enum EnterpriseSyncError {
Io(#[from] std::io::Error),
}
+impl EnterpriseSyncError {
+ fn is_read_refresh(&self) -> bool {
+ let (operation, cause) = match self {
+ Self::Storage { operation, cause } => (*operation, cause),
+ Self::LocalApi(cause) => ("local read", cause),
+ _ => return false,
+ };
+ let pending = screenpipe_db::storage::is_revision_change_message(cause);
+ if pending {
+ // Normal support logs explain pending work without reporting an
+ // upload error or claiming that this stream was acknowledged.
+ info!(
+ operation,
+ cause,
+ retry_seconds = SYNC_INTERVAL.as_secs(),
+ "enterprise sync: selection refresh pending; checkpoint retained"
+ );
+ }
+ pending
+ }
+}
+
// ─── Pure logic: UTF-8-safe truncation ──────────────────────────────────────
/// Truncate `s` to at most `max_bytes`, rounding the cut DOWN to the nearest
@@ -864,6 +896,26 @@ async fn run_one_sync_inner(
http: &reqwest::Client,
include_snapshot: bool,
backfill: Option<&backfill::BackfillRequest>,
+) -> Result<SyncTickReport, EnterpriseSyncError> {
+ match run_one_sync_page(cfg, cursor, local, http, include_snapshot, backfill).await {
+ Err(error) if error.is_read_refresh() => {
+ debug!("enterprise sync: source changed during read; checkpoint retained for the next cycle");
+ Ok(SyncTickReport {
+ pending_read: true,
+ ..Default::default()
+ })
+ }
+ result => result,
+ }
+}
+
+async fn run_one_sync_page(
+ cfg: &EnterpriseSyncConfig,
+ cursor: &mut Cursor,
+ local: &dyn LocalApiClient,
+ http: &reqwest::Client,
+ include_snapshot: bool,
+ backfill: Option<&backfill::BackfillRequest>,
) -> Result<SyncTickReport, EnterpriseSyncError> {
if let EnterpriseUploadMode::Blocked(reason) = &cfg.upload_mode {
return Err(EnterpriseSyncError::Configuration(reason.clone()));
@@ -967,25 +1019,42 @@ async fn run_one_sync_inner(
streams = request.restrict_streams(streams)?;
}
+ let mut pending_read = false;
let mut frames = if streams.frames {
- local
+ match local
.fetch_frames_since(
cursor.last_frame_ts.as_deref(),
cursor.boundary.frames,
FRAME_PAGE_LIMIT,
)
- .await?
+ .await
+ {
+ Ok(rows) => rows,
+ Err(error) if error.is_read_refresh() => {
+ pending_read = true;
+ Vec::new()
+ }
+ Err(error) => return Err(error),
+ }
} else {
Vec::new()
};
let mut audio = if streams.audio {
- local
+ match local
.fetch_audio_since(
cursor.last_audio_ts.as_deref(),
cursor.boundary.audio,
PAGE_LIMIT,
)
- .await?
+ .await
+ {
+ Ok(rows) => rows,
+ Err(error) if error.is_read_refresh() => {
+ pending_read = true;
+ Vec::new()
+ }
+ Err(error) => return Err(error),
+ }
} else {
Vec::new()
};
@@ -998,6 +1067,10 @@ async fn run_one_sync_inner(
.await
{
Ok(rows) => rows,
+ Err(e) if e.is_read_refresh() => {
+ pending_read = true;
+ Vec::new()
+ }
Err(e) => {
if backfill.is_some() {
return Err(e);
@@ -1013,7 +1086,7 @@ async fn run_one_sync_inner(
// not also send the legacy thumbnail stream, even when a cached policy
// enables it. Write-only archives cannot use that image channel; retain
// one original-quality snapshot in their direct-to-storage JSONL instead.
- let snapshots: Vec<SnapshotRow> = if streams.snapshots
+ let mut snapshots: Vec<SnapshotRow> = if streams.snapshots
&& streams.frame_images == crate::enterprise_policy::FrameImagesMode::All
&& include_snapshot
&& matches!(cfg.upload_mode, EnterpriseUploadMode::DirectWriteOnly(_))
@@ -1021,6 +1094,10 @@ async fn run_one_sync_inner(
match local.fetch_latest_snapshot().await {
Ok(Some(s)) => vec![s],
Ok(None) => Vec::new(),
+ Err(e) if e.is_read_refresh() => {
+ pending_read = true;
+ Vec::new()
+ }
Err(e) => {
warn!("enterprise sync: snapshot fetch failed (skipping): {}", e);
Vec::new()
@@ -1042,6 +1119,10 @@ async fn run_one_sync_inner(
.await
{
Ok(rows) => rows,
+ Err(e) if e.is_read_refresh() => {
+ pending_read = true;
+ Vec::new()
+ }
Err(e) => {
if backfill.is_some() {
return Err(e);
@@ -1063,6 +1144,10 @@ async fn run_one_sync_inner(
.await
{
Ok(rows) => rows,
+ Err(error) if error.is_read_refresh() => {
+ pending_read = true;
+ Vec::new()
+ }
Err(error) => {
if backfill.is_some() {
return Err(error);
@@ -1077,13 +1162,6 @@ async fn run_one_sync_inner(
} else {
Vec::new()
};
- if streams.feedback == crate::enterprise_policy::FeedbackSyncMode::Ratings {
- for row in &mut feedback {
- row.comment = None;
- row.snapshot = None;
- row.context = serde_json::json!({});
- }
- }
// Parsed app data is a separate privacy-sensitive stream. It is best
// effort because parser support is optional and older local servers do
// not expose content_type=parsed.
@@ -1097,6 +1175,10 @@ async fn run_one_sync_inner(
.await
{
Ok(rows) => rows,
+ Err(e) if e.is_read_refresh() => {
+ pending_read = true;
+ Vec::new()
+ }
Err(e) => {
if backfill.is_some() {
return Err(e);
@@ -1108,12 +1190,16 @@ async fn run_one_sync_inner(
} else {
Vec::new()
};
- let activities = if streams.activities {
+ let mut activities = if streams.activities {
match local
.fetch_activities_since(cursor.boundary.activity_ts.as_deref())
.await
{
Ok(rows) => rows,
+ Err(error) if error.is_read_refresh() => {
+ pending_read = true;
+ Vec::new()
+ }
Err(error) => {
warn!(
"enterprise sync: activity fetch failed (skipping): {}",
@@ -1145,7 +1231,10 @@ async fn run_one_sync_inner(
&& feedback.is_empty()
{
debug!("enterprise sync: nothing new since last tick");
- return Ok(SyncTickReport::default());
+ return Ok(SyncTickReport {
+ pending_read,
+ ..Default::default()
+ });
}
// A server restart/database replacement during the reads must not label a
@@ -1156,24 +1245,6 @@ async fn run_one_sync_inner(
));
}
- let mut body = screenpipe_telemetry_wire::build_jsonl_with_semantic_streams(
- &cfg.device_id,
- &cfg.device_label,
- &frames,
- &parsed,
- &activities,
- &audio,
- &ui,
- &snapshots,
- &memories,
- );
- body.extend_from_slice(&screenpipe_telemetry_wire::build_feedback_jsonl(
- &cfg.device_id,
- &cfg.device_label,
- &feedback,
- ));
- let bytes = body.len();
-
// The history cursor is the retry checkpoint for images as well as text.
// Store this page's images first: the existing server sees them as already
// resolved, so it need not accumulate another historical request backlog.
@@ -1228,25 +1299,123 @@ async fn run_one_sync_inner(
|row| &row.timestamp,
);
- if !backfill.is_some_and(|request| request.images_only) {
+ let selected = SyncTickReport {
+ frames: frames.len(),
+ audio: audio.len(),
+ ui: ui.len(),
+ memories: memories.len(),
+ parsed: parsed.len(),
+ feedback: feedback.len(),
+ ..Default::default()
+ };
+ // Keep the original scan boundary when its tail is deleted, and recompute
+ // timestamp offsets against the current rows rather than stale positions.
+ if let Some(export) = export
+ .as_ref()
+ .filter(|_| !matches!(cfg.upload_mode, EnterpriseUploadMode::HostedIngest))
+ {
+ let filtered = export
+ .filter(
+ &mut ExportRows {
+ frames: &mut frames,
+ audio: &mut audio,
+ ui: &mut ui,
+ snapshots: &mut snapshots,
+ memories: &mut memories,
+ parsed: &mut parsed,
+ activities: &mut activities,
+ feedback: &mut feedback,
+ },
+ local,
+ )
+ .await?;
+ filtered.apply(&mut next_cursor);
+ }
+
+ if streams.feedback == crate::enterprise_policy::FeedbackSyncMode::Ratings {
+ for row in &mut feedback {
+ row.comment = None;
+ row.snapshot = None;
+ row.context = serde_json::json!({});
+ }
+ }
+ // Filtering may read the running local server again. Preserve the source
+ // identity check across that extra read boundary as well.
+ if cursor.source_id.is_some() && local.upload_source_id().await? != cfg.device_id {
+ return Err(EnterpriseSyncError::Configuration(
+ "database changed during export filtering; retrying".into(),
+ ));
+ }
+
+ let mut body = screenpipe_telemetry_wire::build_jsonl_with_semantic_streams(
+ &cfg.device_id,
+ &cfg.device_label,
+ &frames,
+ &parsed,
+ &activities,
+ &audio,
+ &ui,
+ &snapshots,
+ &memories,
+ );
+ body.extend_from_slice(&screenpipe_telemetry_wire::build_feedback_jsonl(
+ &cfg.device_id,
+ &cfg.device_label,
+ &feedback,
+ ));
+ let bytes = body.len();
+
+ let mut uploaded = None;
+ if !body.is_empty() && !backfill.is_some_and(|request| request.images_only) {
match &cfg.upload_mode {
EnterpriseUploadMode::HostedIngest => {
- for request_body in split_jsonl_requests(body, HOSTED_INGEST_REQUEST_BYTES) {
- let admission = match &export {
- Some(token) => token.admit().await?,
- None => None,
- };
- let upload = post_jsonl_with_identity(
- http,
- &cfg.ingest_url,
- &cfg.license_key,
+ let mut sent = SyncTickReport::default();
+ let mut pending = std::collections::VecDeque::from(split_jsonl_requests(
+ body,
+ HOSTED_INGEST_REQUEST_BYTES,
+ ));
+ while let Some(request_body) = pending.pop_front() {
+ let (request_body, report) = export_filter::filter_request(
+ cfg,
+ export.as_deref(),
+ local,
request_body,
- cfg.stable_device_id.as_deref(),
- backfill.is_some(),
- );
- drop(admission);
- upload.await?;
+ &mut next_cursor,
+ streams.feedback == crate::enterprise_policy::FeedbackSyncMode::Ratings,
+ )
+ .await?;
+ if cursor.source_id.is_some()
+ && local.upload_source_id().await? != cfg.device_id
+ {
+ return Err(EnterpriseSyncError::Configuration(
+ "database changed during export filtering; retrying".into(),
+ ));
+ }
+ // Refreshed contents can grow. Queue the smaller pieces
+ // through this same filter so every actual HTTP request
+ // has a current selection, including after a network wait.
+ let mut pieces =
+ split_jsonl_requests(request_body, HOSTED_INGEST_REQUEST_BYTES);
+ if pieces.len() > 1 {
+ for piece in pieces.into_iter().rev() {
+ pending.push_front(piece);
+ }
+ continue;
+ }
+ if let Some(request_body) = pieces.pop() {
+ post_jsonl_with_identity(
+ http,
+ &cfg.ingest_url,
+ &cfg.license_key,
+ request_body,
+ cfg.stable_device_id.as_deref(),
+ backfill.is_some(),
+ )
+ .await?;
+ }
+ sent.add_assign(&report);
}
+ uploaded = Some(sent);
}
EnterpriseUploadMode::DirectWriteOnly(direct) => {
let counts = DirectUploadRecordCounts {
@@ -1259,10 +1428,6 @@ async fn run_one_sync_inner(
memories: memories.len(),
feedback: feedback.len(),
};
- let admission = match &export {
- Some(token) => token.admit().await?,
- None => None,
- };
let upload = upload_direct_write_only_batch(
http,
cfg,
@@ -1271,7 +1436,6 @@ async fn run_one_sync_inner(
counts,
enterprise_upload::direct_upload_cursors(&next_cursor),
);
- drop(admission);
upload.await?;
}
EnterpriseUploadMode::DirectReadable(direct) => {
@@ -1285,10 +1449,6 @@ async fn run_one_sync_inner(
memories: memories.len(),
feedback: feedback.len(),
};
- let admission = match &export {
- Some(token) => token.admit().await?,
- None => None,
- };
let upload = upload_direct_readable_batch(
http,
cfg,
@@ -1298,7 +1458,6 @@ async fn run_one_sync_inner(
enterprise_upload::direct_upload_cursors(&next_cursor),
backfill.is_some(),
);
- drop(admission);
upload.await?;
}
EnterpriseUploadMode::Blocked(reason) => {
@@ -1306,9 +1465,27 @@ async fn run_one_sync_inner(
}
}
}
+ let mut report = uploaded.unwrap_or(SyncTickReport {
+ frames: frames.len(),
+ parsed: parsed.len(),
+ activities: activities.len(),
+ audio: audio.len(),
+ ui: ui.len(),
+ snapshots: snapshots.len(),
+ memories: memories.len(),
+ feedback: feedback.len(),
+ bytes,
+ ..Default::default()
+ });
+ report.scanned_full_page = selected.may_have_more();
+ report.pending_read = pending_read;
if backfill.is_some() {
- let records =
- frames.len() + audio.len() + ui.len() + parsed.len() + memories.len() + feedback.len();
+ let records = report.frames
+ + report.audio
+ + report.ui
+ + report.parsed
+ + report.memories
+ + report.feedback;
next_cursor.boundary.backfill_records =
Some(cursor.boundary.backfill_records.unwrap_or(0) + records as u64);
if let Some(images) = images {
@@ -1322,17 +1499,7 @@ async fn run_one_sync_inner(
*cursor = next_cursor;
cursor.save(&cfg.cursor_path)?;
- Ok(SyncTickReport {
- frames: frames.len(),
- parsed: parsed.len(),
- activities: activities.len(),
- audio: audio.len(),
- ui: ui.len(),
- snapshots: snapshots.len(),
- memories: memories.len(),
- feedback: feedback.len(),
- bytes,
- })
+ Ok(report)
}
fn advance_timestamp_boundary<T>(
@@ -1370,6 +1537,10 @@ pub struct SyncTickReport {
pub memories: usize,
pub feedback: usize,
pub bytes: usize,
+ /// Filtering must not make a full scanned page look like the end of history.
+ pub scanned_full_page: bool,
+ /// At least one source read is pending; retain its checkpoint for a later cycle.
+ pub pending_read: bool,
}
impl SyncTickReport {
@@ -1377,7 +1548,9 @@ impl SyncTickReport {
/// behind it. Point-in-time snapshots are deliberately excluded.
pub fn may_have_more(&self) -> bool {
let limit = PAGE_LIMIT as usize;
- self.frames >= FRAME_PAGE_LIMIT as usize
+ self.pending_read
+ || self.scanned_full_page
+ || self.frames >= FRAME_PAGE_LIMIT as usize
|| self.parsed >= limit
|| self.audio >= limit
|| self.ui >= limit
@@ -1388,12 +1561,15 @@ impl SyncTickReport {
fn add_assign(&mut self, other: &Self) {
self.frames += other.frames;
self.parsed += other.parsed;
+ self.activities += other.activities;
self.audio += other.audio;
self.ui += other.ui;
self.snapshots += other.snapshots;
self.memories += other.memories;
self.feedback += other.feedback;
self.bytes += other.bytes;
+ self.scanned_full_page |= other.scanned_full_page;
+ self.pending_read |= other.pending_read;
}
}
@@ -1430,7 +1606,7 @@ pub async fn run_sync_burst(
burst.total.add_assign(&page);
burst.pages += 1;
- if !more_pending {
+ if !more_pending || page.pending_read {
break;
}
}
@@ -3730,6 +3906,226 @@ pub(crate) mod tests {
}
}
+ #[tokio::test]
+ async fn later_upload_requests_filter_deletions_and_redactions_after_an_earlier_ack() {
+ let _guard = crate::enterprise_policy::sync_streams_test_lock();
+ let _restore = RestoreDefaultSyncStreams;
+ crate::enterprise_policy::set_sync_streams(
+ true,
+ false,
+ false,
+ false,
+ false,
+ false,
+ "off".into(),
+ "off".into(),
+ );
+ struct CurrentRows(Arc<Mutex<Vec<FrameRow>>>);
+ struct CurrentFilter;
+ #[async_trait::async_trait]
+ impl ExportFilter for CurrentFilter {
+ async fn filter(
+ &self,
+ rows: &mut ExportRows<'_>,
+ local: &dyn LocalApiClient,
+ ) -> Result<ExportFilterReport, EnterpriseSyncError> {
+ rows.refresh(local).await
+ }
+ }
+ #[async_trait::async_trait]
+ impl LocalApiClient for CurrentRows {
+ async fn begin_export(
+ &self,
+ ) -> Result<Option<Box<dyn ExportFilter>>, EnterpriseSyncError> {
+ Ok(Some(Box::new(CurrentFilter)))
+ }
+ async fn fetch_frames_since(
+ &self,
+ since: Option<&str>,
+ offset: u32,
+ limit: u32,
+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
+ Ok(self
+ .0
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|row| since.is_none_or(|since| row.timestamp.as_str() >= since))
+ .skip(offset as usize)
+ .take(limit as usize)
+ .cloned()
+ .collect())
+ }
+ async fn fetch_audio_since(
+ &self,
+ _: Option<&str>,
+ _: u32,
+ _: u32,
+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
+ Ok(Vec::new())
+ }
+ }
+ let ts = "2026-08-04T00:01:00Z";
+ // Two records exceed the actual transport target, so each selected
+ // record initially occupies its own HTTP request.
+ let text = "x".repeat(HOSTED_INGEST_REQUEST_BYTES / 2);
+ let rows = Arc::new(Mutex::new(
+ (1..=3)
+ .map(|id| frame(id, ts, "Arc", &text))
+ .collect::<Vec<_>>(),
+ ));
+ let server = wiremock::MockServer::start().await;
+ let changed = rows.clone();
+ let responses = Arc::new(AtomicUsize::new(0));
+ wiremock::Mock::given(wiremock::matchers::path("/ingest"))
+ .respond_with(move |_: &wiremock::Request| {
+ if responses.fetch_add(1, Ordering::SeqCst) == 0 {
+ let mut rows = changed.lock().unwrap();
+ rows.retain(|row| row.frame_id != 2);
+ rows.iter_mut().find(|row| row.frame_id == 3).unwrap().text =
+ Some("[REDACTED]".into());
+ rows.push(frame(4, ts, "Arc", "new capture"));
+ }
+ wiremock::ResponseTemplate::new(200)
+ })
+ .expect(3)
+ .mount(&server)
+ .await;
+ let dir = TempDir::new().unwrap();
+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));
+ let mut cursor = initialized_cursor();
+ let local = CurrentRows(rows);
+ let http = reqwest::Client::new();
+ let report = run_one_sync(&cfg, &mut cursor, &local, &http)
+ .await
+ .unwrap();
+ assert_eq!(report.frames, 2);
+ assert_eq!(cursor.boundary.frames, 2);
+ let mut restarted = Cursor::load(&cfg.cursor_path);
+ assert_eq!(
+ run_one_sync(&cfg, &mut restarted, &local, &http)
+ .await
+ .unwrap()
+ .frames,
+ 1
+ );
+ let requests = server.received_requests().await.unwrap();
+ let records: Vec<serde_json::Value> = requests
+ .iter()
+ .map(|request| serde_json::from_slice(&request.body).unwrap())
+ .collect();
+ assert_eq!(
+ records
+ .iter()
+ .map(|record| record["frame_id"].as_i64().unwrap())
+ .collect::<Vec<_>>(),
+ vec![1, 3, 4]
+ );
+ assert_eq!(records[1]["text"], "[REDACTED]");
+ assert_eq!(records[2]["text"], "new capture");
+ server.verify().await;
+ }
+
+ #[tokio::test]
+ async fn changed_read_keeps_only_that_stream_pending_and_recovers_after_restart() {
+ let _guard = crate::enterprise_policy::sync_streams_test_lock();
+ let _restore = RestoreDefaultSyncStreams;
+ crate::enterprise_policy::set_sync_streams(
+ true,
+ false,
+ true,
+ false,
+ false,
+ false,
+ "off".into(),
+ "off".into(),
+ );
+ struct ChangingLocal(AtomicUsize);
+ #[async_trait::async_trait]
+ impl LocalApiClient for ChangingLocal {
+ async fn fetch_frames_since(
+ &self,
+ _: Option<&str>,
+ _: u32,
+ _: u32,
+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
+ if self.0.fetch_add(1, Ordering::SeqCst) == 0 {
+ return Err(EnterpriseSyncError::LocalApi(
+ "read frame AX payloads: frame storage: read revision changed; retry the complete query".into()));
+ }
+ Ok(vec![FrameRow {
+ frame_id: 1,
+ timestamp: "2026-08-04T00:01:00Z".into(),
+ app_name: None,
+ window_name: None,
+ browser_url: None,
+ text: Some("retained frame".into()),
+ accessibility_tree: None,
+ }])
+ }
+ async fn fetch_audio_since(
+ &self,
+ _: Option<&str>,
+ offset: u32,
+ _: u32,
+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
+ if offset > 0 {
+ return Ok(Vec::new());
+ }
+ Ok(vec![AudioRow {
+ transcription_id: 1,
+ timestamp: "2026-08-04T00:01:00Z".into(),
+ transcription: "unaffected audio".into(),
+ speaker: None,
+ device: None,
+ }])
+ }
+ }
+ let server = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::path("/ingest"))
+ .respond_with(wiremock::ResponseTemplate::new(200))
+ .expect(2)
+ .mount(&server)
+ .await;
+ let dir = TempDir::new().unwrap();
+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));
+ let mut cursor = initialized_cursor();
+ cursor.save(&cfg.cursor_path).unwrap();
+ let before_frames = cursor.last_frame_ts.clone();
+ let local = ChangingLocal(AtomicUsize::new(0));
+ let http = reqwest::Client::new();
+ let report = run_sync_burst(&cfg, &mut cursor, &local, &http)
+ .await
+ .unwrap();
+ assert!(report.total.pending_read);
+ assert!(
+ report.total.may_have_more(),
+ "pending history must not be marked complete"
+ );
+ assert_eq!(report.total.audio, 1);
+ assert_eq!(cursor.last_frame_ts, before_frames);
+ assert_eq!(
+ cursor.last_audio_ts.as_deref(),
+ Some("2026-08-04T00:01:00Z")
+ );
+ assert!(!cfg
+ .cursor_path
+ .with_file_name(diagnostics::LOG_NAME)
+ .exists());
+ let mut restarted = Cursor::load(&cfg.cursor_path);
+ let report = run_one_sync(&cfg, &mut restarted, &local, &http)
+ .await
+ .unwrap();
+ assert!(!report.pending_read);
+ assert_eq!(report.frames, 1);
+ assert_eq!(report.audio, 0, "acknowledged audio must not be replayed");
+ assert_eq!(
+ Cursor::load(&cfg.cursor_path).last_frame_ts.as_deref(),
+ Some("2026-08-04T00:01:00Z")
+ );
+ server.verify().await;
+ }
+
/// Exercise the production worker, including its actual sleeps and
/// persisted checkpoint, rather than only checking a delay constant.
#[tokio::test(flavor = "current_thread")]
@@ -3819,12 +4215,12 @@ pub(crate) mod tests {
reply
.send(Err(EnterpriseSyncError::Storage {
operation: if attempt % 2 == 0 {
- "admit export"
+ "filter export"
} else {
"prepare export"
},
cause: if attempt % 2 == 0 {
- "frame storage: read revision changed; retry the complete query"
+ "database disk image is malformed"
} else {
"frame storage: payload read deadline exceeded"
}
@@ -3908,7 +4304,7 @@ pub(crate) mod tests {
let logged = std::fs::read_to_string(log_path).unwrap();
assert!(logged.contains("enterprise upload storage prepare export failed"));
assert!(logged.contains("frame storage: payload read deadline exceeded"));
- assert!(logged.contains("frame storage: read revision changed"));
+ assert!(logged.contains("database disk image is malformed"));
assert!(logged.contains("pending work retained, retrying in 300s"));
assert!(!logged.contains("backing off 3600s"));
String::from_utf8(diagnostic.body.clone()).unwrap()
@@ -4465,10 +4861,22 @@ pub(crate) mod tests {
struct UnavailableTree;
#[async_trait::async_trait]
impl LocalApiClient for UnavailableTree {
- async fn fetch_frames_since(&self, _: Option<&str>, _: u32, _: u32) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
- Err(EnterpriseSyncError::LocalApi("read frame AX payloads: unavailable".into()))
+ async fn fetch_frames_since(
+ &self,
+ _: Option<&str>,
+ _: u32,
+ _: u32,
+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
+ Err(EnterpriseSyncError::LocalApi(
+ "read frame AX payloads: unavailable".into(),
+ ))
}
- async fn fetch_audio_since(&self, _: Option<&str>, _: u32, _: u32) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
+ async fn fetch_audio_since(
+ &self,
+ _: Option<&str>,
+ _: u32,
+ _: u32,
+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
panic!("failed frame read must abort before other streams")
}
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
index 0c127d04d..7cac9d0be 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
@@ -193,6 +193,23 @@ mod imp {
updated_at: String,
}
+ async fn local_api_error(response: reqwest::Response, url: &str) -> EnterpriseSyncError {
+ let status = response.status();
+ let cause = response
+ .json::<serde_json::Value>()
+ .await
+ .ok()
+ .and_then(|body| {
+ body.get("error")
+ .and_then(|v| v.as_str())
+ .map(str::to_owned)
+ });
+ EnterpriseSyncError::LocalApi(match cause {
+ Some(cause) => format!("GET {url} -> {status}: {cause}"),
+ None => format!("GET {url} -> {status}"),
+ })
+ }
+
struct StorageExport {
db: Arc<screenpipe_db::DatabaseManager>,
token: screenpipe_db::storage::StorageReadToken,
@@ -214,17 +231,33 @@ mod imp {
}
#[async_trait::async_trait]
- impl crate::enterprise::sync::ExportAdmission for StorageExport {
- async fn admit(
+ impl crate::enterprise::sync::ExportFilter for StorageExport {
+ async fn filter(
&self,
- ) -> Result<Option<tokio::sync::OwnedMutexGuard<()>>, EnterpriseSyncError> {
- self.token
- .admit(&self.db.pool)
- .await
- .map_err(|error| EnterpriseSyncError::Storage {
- operation: "admit export",
+ rows: &mut ee_sync::ExportRows<'_>,
+ local: &dyn LocalApiClient,
+ ) -> Result<ee_sync::ExportFilterReport, EnterpriseSyncError> {
+ // SQLite has no hybrid revocation counter. Validate its selected
+ // rows directly instead of treating a no-op token as freshness.
+ if self.db.storage_mode() == screenpipe_db::storage::StorageMode::Sqlite {
+ return rows.refresh(local).await;
+ }
+ match self.token.admit(&self.db.pool).await {
+ Ok(guard) => {
+ drop(guard);
+ Ok(Default::default())
+ }
+ Err(error) if screenpipe_db::storage::is_revision_change(&error) => {
+ // A changed revision is a request to filter the selected
+ // rows, not an upload failure. Reads return the current
+ // contents of survivors, including applied redactions.
+ rows.refresh(local).await
+ }
+ Err(error) => Err(EnterpriseSyncError::Storage {
+ operation: "filter export",
cause: error.to_string(),
- })
+ }),
+ }
}
}
@@ -232,7 +265,7 @@ mod imp {
impl LocalApiClient for ScreenpipeLocalClient {
async fn begin_export(
&self,
- ) -> Result<Option<Box<dyn crate::enterprise::sync::ExportAdmission>>, EnterpriseSyncError>
+ ) -> Result<Option<Box<dyn crate::enterprise::sync::ExportFilter>>, EnterpriseSyncError>
{
let state = self.app.state::<crate::recording::RecordingState>();
let db = state
@@ -352,11 +385,7 @@ mod imp {
.await
.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;
if !resp.status().is_success() {
- return Err(EnterpriseSyncError::LocalApi(format!(
- "GET {} -> {}",
- url,
- resp.status()
- )));
+ return Err(local_api_error(resp, &url).await);
}
let body: LocalSearchResponse = resp
.json()
@@ -424,11 +453,7 @@ mod imp {
.await
.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;
if !resp.status().is_success() {
- return Err(EnterpriseSyncError::LocalApi(format!(
- "GET {} -> {}",
- url,
- resp.status()
- )));
+ return Err(local_api_error(resp, &url).await);
}
let body: LocalSearchResponse = resp
.json()
@@ -469,11 +494,7 @@ mod imp {
.await
.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;
if !resp.status().is_success() {
- return Err(EnterpriseSyncError::LocalApi(format!(
- "GET {} -> {}",
- url,
- resp.status()
- )));
+ return Err(local_api_error(resp, &url).await);
}
let body: LocalSearchResponse = resp
.json()
@@ -554,11 +575,7 @@ mod imp {
.await
.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;
if !resp.status().is_success() {
- return Err(EnterpriseSyncError::LocalApi(format!(
- "GET {} -> {}",
- url,
- resp.status()
- )));
+ return Err(local_api_error(resp, &url).await);
}
let body: LocalSearchResponse = resp
.json()
@@ -629,11 +646,7 @@ mod imp {
.await
.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;
if !resp.status().is_success() {
- return Err(EnterpriseSyncError::LocalApi(format!(
- "GET {} -> {}",
- search_url,
- resp.status()
- )));
+ return Err(local_api_error(resp, &search_url).await);
}
let body: LocalSearchResponse = resp
.json()
@@ -738,11 +751,7 @@ mod imp {
.await
.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;
if !resp.status().is_success() {
- return Err(EnterpriseSyncError::LocalApi(format!(
- "GET {} -> {}",
- url,
- resp.status()
- )));
+ return Err(local_api_error(resp, &url).await);
}
let body: LocalMemoriesResponse = resp
.json()
@@ -787,11 +796,7 @@ mod imp {
.await
.map_err(|error| EnterpriseSyncError::LocalApi(error.to_string()))?;
if !resp.status().is_success() {
- return Err(EnterpriseSyncError::LocalApi(format!(
- "GET {} -> {}",
- url,
- resp.status()
- )));
+ return Err(local_api_error(resp, &url).await);
}
let body: LocalFeedbackResponse = resp
.json()
@@ -2317,19 +2322,35 @@ mod imp {
}
#[tokio::test]
- async fn export_upload_proceeds_during_capture_and_rejects_deleted_data() {
+ async fn export_local_read_change_preserves_the_source_cause() {
+ let server = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::path("/search"))
+ .respond_with(wiremock::ResponseTemplate::new(503).set_body_json(
+ serde_json::json!({"error":"frame storage: read revision changed; retry the complete query"})
+ )).mount(&server).await;
+ let url = format!("{}/search", server.uri());
+ let response = reqwest::Client::new().get(&url).send().await.unwrap();
+ let error = super::local_api_error(response, &url).await;
+ assert!(screenpipe_db::storage::is_revision_change_message(
+ &error.to_string()
+ ));
+ }
+
+ #[tokio::test]
+ async fn export_upload_filters_deleted_rows_and_refreshes_redactions() {
use super::{AudioRow, EnterpriseSyncError, FrameRow, LocalApiClient, StorageExport};
- use crate::enterprise::sync::{self, ExportAdmission};
+ use crate::enterprise::sync::{self, ExportFilter};
use std::sync::Arc;
struct CapturingLocal {
db: Arc<screenpipe_db::DatabaseManager>,
- delete_selected: bool,
+ change: u8,
+ selected: std::sync::atomic::AtomicBool,
}
#[async_trait::async_trait]
impl LocalApiClient for CapturingLocal {
async fn begin_export(
&self,
- ) -> Result<Option<Box<dyn ExportAdmission>>, EnterpriseSyncError> {
+ ) -> Result<Option<Box<dyn ExportFilter>>, EnterpriseSyncError> {
Ok(Some(Box::new(StorageExport::begin(self.db.clone()).await?)))
}
async fn fetch_frames_since(
@@ -2338,30 +2359,52 @@ mod imp {
_: u32,
_: u32,
) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
+ let first = !self
+ .selected
+ .swap(true, std::sync::atomic::Ordering::SeqCst);
+ let ids = if first { vec![1, 2] } else { vec![1, 2, 3] };
let payloads = self
.db
- .frame_payloads(&[1], screenpipe_db::storage::Projection::All)
+ .frame_payloads(&ids, screenpipe_db::storage::Projection::All)
.await
.unwrap();
- let text = payloads.get(&1).unwrap().full_text.clone();
- // Capture continues after selecting the outgoing page.
- self.db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-10-09','new capture')").await.unwrap();
- if self.delete_selected {
- self.db
- .execute_raw_sql_write("DELETE FROM frames WHERE id=1")
- .await
- .unwrap();
+ let rows = payloads
+ .into_iter()
+ .map(|(id, payload)| FrameRow {
+ accessibility_tree: None,
+ frame_id: id,
+ timestamp: "2026-10-09T00:00:00Z".into(),
+ app_name: None,
+ window_name: None,
+ browser_url: None,
+ text: payload.full_text,
+ })
+ .collect();
+ if first {
+ // Recording remains durable while this outgoing page is selected.
+ self.db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-10-09','new capture')").await.unwrap();
+ match self.change {
+ 1 => {
+ self.db
+ .execute_raw_sql_write("DELETE FROM frames WHERE id=1")
+ .await
+ .unwrap();
+ }
+ 2 => {
+ self.db.execute_raw_sql_write("UPDATE frames SET full_text='[REDACTED]',full_text_redacted_at=1 WHERE id=1").await.unwrap();
+ }
+ 3 => {
+ self.db
+ .execute_raw_sql_write("DELETE FROM frames WHERE id IN (1,2)")
+ .await
+ .unwrap();
+ }
+ _ => {}
+ }
}
- Ok(vec![FrameRow {
- accessibility_tree: None,
- frame_id: 1,
- timestamp: "2026-10-09T00:00:00Z".into(),
- app_name: None,
- window_name: None,
- browser_url: None,
- text,
- }])
+ Ok(rows)
}
+
async fn fetch_audio_since(
&self,
_: Option<&str>,
@@ -2371,64 +2414,74 @@ mod imp {
Ok(Vec::new())
}
}
- for delete_selected in [false, true] {
+ for (change, hybrid) in (0..4).flat_map(|change| [(change, false), (change, true)]) {
let dir = tempfile::tempdir().unwrap();
- let db = Arc::new(
+ let db = Arc::new(if hybrid {
+ // Synthetic export fixture; disk admission is covered by
+ // the DB tests, independently of developer disk capacity.
+ let mut options = screenpipe_db::storage::MigrationOptions::default();
+ options.budget.disk_reserve_bytes = 0;
screenpipe_db::DatabaseManager::new_hybrid(
dir.path(),
Default::default(),
+ options,
+ )
+ .await
+ .unwrap()
+ } else {
+ screenpipe_db::DatabaseManager::new(
+ dir.path().join("db.sqlite").to_str().unwrap(),
Default::default(),
)
.await
- .unwrap(),
- );
- db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-09','pending upload')").await.unwrap();
+ .unwrap()
+ });
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-09','pending upload'),(2,'2026-10-09','surviving record')").await.unwrap();
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::path("/ingest"))
.respond_with(wiremock::ResponseTemplate::new(200))
- .expect(if delete_selected { 0 } else { 1 })
+ .expect(if change == 3 { 0 } else { 1 })
.mount(&server)
.await;
let cfg = sync::tests::test_cfg(&dir, format!("{}/ingest", server.uri()));
let mut cursor = sync::Cursor::default();
let local = CapturingLocal {
db: db.clone(),
- delete_selected,
+ change,
+ selected: std::sync::atomic::AtomicBool::new(false),
};
let result =
sync::run_one_sync(&cfg, &mut cursor, &local, &reqwest::Client::new()).await;
- if delete_selected {
- let error = result.unwrap_err();
- assert!(matches!(
- &error,
- EnterpriseSyncError::Storage {
- operation: "admit export",
- ..
- }
- ));
- assert!(error.to_string().contains("read revision changed"));
+ let expected = match change {
+ 1 => 1,
+ 3 => 0,
+ _ => 2,
+ };
+ assert_eq!(result.unwrap().frames, expected);
+ assert_eq!(
+ sync::Cursor::load(&cfg.cursor_path)
+ .last_frame_ts
+ .as_deref(),
+ Some("2026-10-09T00:00:00Z")
+ );
+ let requests = server.received_requests().await.unwrap();
+ if change != 3 {
+ let body = String::from_utf8_lossy(&requests[0].body);
+ assert!(body.contains("surviving record"));
assert!(
- !cfg.cursor_path.exists(),
- "revoked data must not be checkpointed"
- );
- } else {
- assert_eq!(result.unwrap().frames, 1);
- assert_eq!(
- sync::Cursor::load(&cfg.cursor_path)
- .last_frame_ts
- .as_deref(),
- Some("2026-10-09T00:00:00Z")
+ !body.contains("new capture"),
+ "new rows belong to the next page"
);
- let requests = server.received_requests().await.unwrap();
- assert!(String::from_utf8_lossy(&requests[0].body).contains("pending upload"));
+ assert_eq!(body.contains("pending upload"), change == 0);
+ assert_eq!(body.contains("[REDACTED]"), change == 2);
}
server.verify().await;
let captured = db
- .frame_payloads(&[2], screenpipe_db::storage::Projection::All)
+ .frame_payloads(&[3], screenpipe_db::storage::Projection::All)
.await
.unwrap();
assert_eq!(
- captured.get(&2).unwrap().full_text.as_deref(),
+ captured.get(&3).unwrap().full_text.as_deref(),
Some("new capture"),
"upload admission must preserve the new recording"
);
diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rs
index 197eaf0bb..348deca51 100644
--- a/crates/screenpipe-db/src/storage/mod.rs
+++ b/crates/screenpipe-db/src/storage/mod.rs
@@ -37,7 +37,7 @@ pub use lifecycle::{
MigrationReport,
};
pub use maintenance::{compact, export_sqlite, recover_interrupted_compaction};
-pub use reader::StorageReadToken;
+pub use reader::{is_revision_change, is_revision_change_message, StorageReadToken};
use serde::{Deserialize, Serialize};
use std::path::{Component, Path, PathBuf};
use std::sync::Arc;
diff --git a/crates/screenpipe-db/src/storage/reader.rs b/crates/screenpipe-db/src/storage/reader.rs
index db13643c0..c3336b9e1 100644
--- a/crates/screenpipe-db/src/storage/reader.rs
+++ b/crates/screenpipe-db/src/storage/reader.rs
@@ -123,6 +123,9 @@ impl DatabaseManager {
/// the duration of a network upload. Final admission still rejects any
/// deletion/privacy change committed since this single-statement read.
pub async fn storage_export_token(&self) -> Result<StorageReadToken, sqlx::Error> {
+ if self.pool.is_closed() {
+ return Err(sqlx::Error::PoolClosed);
+ }
let Some(storage) = &self.storage else {
return self.storage_read_token().await;
};
@@ -287,8 +290,15 @@ impl DatabaseManager {
}
}
-fn is_revision_change(error: &sqlx::Error) -> bool {
- matches!(error, sqlx::Error::Protocol(message) if message.starts_with("frame storage: read revision changed;"))
+const REVISION_CHANGE_PREFIX: &str = "frame storage: read revision changed;";
+
+/// Recognize invalidated reads after the cause crosses an API/error wrapper.
+pub fn is_revision_change_message(message: &str) -> bool {
+ message.contains(REVISION_CHANGE_PREFIX)
+}
+
+pub fn is_revision_change(error: &sqlx::Error) -> bool {
+ matches!(error, sqlx::Error::Protocol(message) if message.starts_with(REVISION_CHANGE_PREFIX))
}
impl HybridStorage {