YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(enterprise): filter changed records without failing uploads (#7667)
Ezra Ellette8001273725a1
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs b/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rsindex 1bdf13ed4..2e08c677d 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs@@ -302,8 +302,9 @@ mod tests {let dir = tempfile::tempdir().unwrap();let report = crate::enterprise::sync::tests::check_sync_retry_failure(dir.path()).await;assert!(report.contains("enterprise upload storage prepare export failed"));+ assert!(report.contains("enterprise upload storage filter export failed"));assert!(report.contains("frame storage: payload read deadline exceeded"));- assert!(report.contains("frame storage: read revision changed"));+ assert!(report.contains("database disk image is malformed"));assert!(report.contains("pending work retained, retrying in 300s"));assert!(!report.contains("backing off 3600s"));}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rsindex efcbccc68..23e45e1f9 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs@@ -1220,7 +1220,7 @@ pub(super) async fn fulfill_requests(.await?;// JPEG pages are heavier than telemetry pages. Yield to live// sync after one acknowledged image page, retaining its cursor.- if complete || cursor.boundary.backfill_images.is_some() {+ if complete || page.pending_read || cursor.boundary.backfill_images.is_some() {break;}}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/export_filter.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/export_filter.rsnew file mode 100644index 000000000..3d3dc6894--- /dev/null+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/export_filter.rs@@ -0,0 +1,515 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use super::*;++/// Absolute offsets recomputed from current source rows, including previously+/// acknowledged rows at the same timestamp. Deleting an earlier row shifts+/// those offsets too; subtracting only deleted outgoing rows would skip data.+#[derive(Default)]+pub struct ExportFilterReport {+ pub frames: Option<(String, u32)>,+ pub audio: Option<(String, u32)>,+ pub ui: Option<(String, u32)>,+ pub memories: Option<(String, u32)>,+ pub parsed: Option<(String, u32)>,+}++impl ExportFilterReport {+ pub(super) fn apply(self, cursor: &mut Cursor) {+ for (current, offset, fresh) in [+ (+ &cursor.last_frame_ts,+ &mut cursor.boundary.frames,+ self.frames,+ ),+ (+ &cursor.last_audio_ts,+ &mut cursor.boundary.audio,+ self.audio,+ ),+ (&cursor.last_ui_ts, &mut cursor.boundary.ui, self.ui),+ (+ &cursor.last_memory_ts,+ &mut cursor.boundary.memories,+ self.memories,+ ),+ (+ &cursor.last_parsed_ts,+ &mut cursor.boundary.parsed,+ self.parsed,+ ),+ ] {+ if let Some((timestamp, count)) = fresh {+ if current.as_deref() == Some(timestamp.as_str()) {+ *offset = count;+ }+ }+ }+ }+}++/// Each HTTP request is filtered again after any preceding network wait.+/// A deletion while an earlier request is in flight must not leak through the+/// already serialized remainder of a large page.+pub(super) async fn filter_request(+ cfg: &EnterpriseSyncConfig,+ export: Option<&dyn ExportFilter>,+ local: &dyn LocalApiClient,+ body: Vec<u8>,+ cursor: &mut Cursor,+ ratings_only: bool,+) -> Result<(Vec<u8>, SyncTickReport), EnterpriseSyncError> {+ let mut frames = Vec::new();+ let mut audio = Vec::new();+ let mut ui = Vec::new();+ let mut snapshots = Vec::new();+ let mut memories = Vec::new();+ let mut parsed = Vec::new();+ let mut activities = Vec::new();+ let mut feedback = Vec::new();+ for line in body+ .split(|byte| *byte == b'\n')+ .filter(|line| !line.is_empty())+ {+ let record: TelemetryRecord = serde_json::from_slice(line)+ .map_err(|e| EnterpriseSyncError::LocalApi(format!("decode prepared export: {e}")))?;+ match record {+ TelemetryRecord::Frame { frame, .. } => frames.push(frame),+ TelemetryRecord::Audio { audio: row, .. } => audio.push(row),+ TelemetryRecord::Ui { ui: row, .. } => ui.push(row),+ TelemetryRecord::Snapshot { snapshot, .. } => snapshots.push(snapshot),+ TelemetryRecord::Memory { memory, .. } => memories.push(memory),+ TelemetryRecord::Parsed { parsed: row, .. } => parsed.push(row),+ TelemetryRecord::Activity { activity, .. } => activities.push(activity),+ TelemetryRecord::Feedback { feedback: row, .. } => feedback.push(row),+ }+ }+ drop(body);+ if let Some(export) = export {+ export+ .filter(+ &mut ExportRows {+ frames: &mut frames,+ audio: &mut audio,+ ui: &mut ui,+ snapshots: &mut snapshots,+ memories: &mut memories,+ parsed: &mut parsed,+ activities: &mut activities,+ feedback: &mut feedback,+ },+ local,+ )+ .await?+ .apply(cursor);+ }+ if ratings_only {+ for row in &mut feedback {+ row.comment = None;+ row.snapshot = None;+ row.context = serde_json::json!({});+ }+ }+ let mut body = screenpipe_telemetry_wire::build_jsonl_with_semantic_streams(+ &cfg.device_id,+ &cfg.device_label,+ &frames,+ &parsed,+ &activities,+ &audio,+ &ui,+ &snapshots,+ &memories,+ );+ body.extend_from_slice(&screenpipe_telemetry_wire::build_feedback_jsonl(+ &cfg.device_id,+ &cfg.device_label,+ &feedback,+ ));+ let report = SyncTickReport {+ frames: frames.len(),+ parsed: parsed.len(),+ activities: activities.len(),+ audio: audio.len(),+ ui: ui.len(),+ snapshots: snapshots.len(),+ memories: memories.len(),+ feedback: feedback.len(),+ bytes: body.len(),+ ..Default::default()+ };+ Ok((body, report))+}++/// Borrow the outgoing page so filtering keeps the original selection and order.+pub struct ExportRows<'a> {+ pub frames: &'a mut Vec<FrameRow>,+ pub audio: &'a mut Vec<AudioRow>,+ pub ui: &'a mut Vec<UiEventRow>,+ pub snapshots: &'a mut Vec<SnapshotRow>,+ pub memories: &'a mut Vec<MemoryRow>,+ pub parsed: &'a mut Vec<ParsedRow>,+ pub activities: &'a mut Vec<ActivityRow>,+ pub feedback: &'a mut Vec<FeedbackRow>,+}++impl ExportRows<'_> {+ pub async fn refresh(+ &mut self,+ local: &dyn LocalApiClient,+ ) -> Result<ExportFilterReport, EnterpriseSyncError> {+ let mut report = ExportFilterReport::default();+ report.frames = filter_current_rows(+ self.frames,+ FRAME_PAGE_LIMIT,+ |r| r.frame_id,+ |r| r.timestamp.as_str(),+ |since, offset, limit| async move {+ local.fetch_frames_since(Some(&since), offset, limit).await+ },+ )+ .await?;+ // The local audio API identifies ordinary audio by chunk + timestamp,+ // and live meeting segments by a negative segment ID.+ report.audio = filter_current_rows(+ self.audio,+ PAGE_LIMIT,+ |r| (r.transcription_id, r.timestamp.clone()),+ |r| r.timestamp.as_str(),+ |since, offset, limit| async move {+ local.fetch_audio_since(Some(&since), offset, limit).await+ },+ )+ .await?;+ report.ui = filter_current_rows(+ self.ui,+ PAGE_LIMIT,+ |r| r.event_id,+ |r| r.timestamp.as_str(),+ |since, offset, limit| async move {+ local+ .fetch_ui_events_since(Some(&since), offset, limit)+ .await+ },+ )+ .await?;+ report.memories = filter_current_rows(+ self.memories,+ PAGE_LIMIT,+ |r| r.memory_id,+ |r| r.created_at.as_str(),+ |since, offset, limit| async move {+ local+ .fetch_memories_since(Some(&since), offset, limit)+ .await+ },+ )+ .await?;+ report.parsed = filter_current_rows(+ self.parsed,+ PAGE_LIMIT,+ |r| r.frame_id,+ |r| r.timestamp.as_str(),+ |since, offset, limit| async move {+ local.fetch_parsed_since(Some(&since), offset, limit).await+ },+ )+ .await?;+ if let Some(first) = self.feedback.first() {+ let mut since = first.updated_at.clone();+ let wanted: std::collections::HashSet<_> = self+ .feedback+ .iter()+ .map(|r| r.feedback_id.clone())+ .collect();+ let mut current = std::collections::HashMap::new();+ let mut after = None;+ loop {+ let page = local+ .fetch_feedback_since(Some(&since), after.as_deref(), PAGE_LIMIT)+ .await?;+ let more = page.len() == PAGE_LIMIT as usize;+ let Some(last) = page.last() else {+ break;+ };+ since = last.updated_at.clone();+ after = Some(last.feedback_id.clone());+ for row in page {+ if wanted.contains(&row.feedback_id) {+ current.insert(row.feedback_id.clone(), row);+ }+ }+ if !more || current.len() == wanted.len() {+ break;+ }+ }+ self.feedback+ .retain_mut(|row| match current.remove(&row.feedback_id) {+ Some(fresh) => {+ *row = fresh;+ true+ }+ None => false,+ });+ }+ if let Some(first) = self.activities.first() {+ let current = local.fetch_activities_since(Some(&first.start_at)).await?;+ let mut current: std::collections::HashMap<_, _> = current+ .into_iter()+ .map(|r| (r.activity_id.clone(), r))+ .collect();+ self.activities+ .retain_mut(|row| match current.remove(&row.activity_id) {+ Some(fresh) => {+ *row = fresh;+ true+ }+ None => false,+ });+ }+ let mut snapshots = Vec::with_capacity(self.snapshots.len());+ for row in self.snapshots.iter() {+ let jpeg = match local.fetch_frame_jpeg(row.frame_id).await {+ Ok(jpeg) => jpeg,+ Err(EnterpriseSyncError::BackfillImages("screenshot_policy_disabled")) => None,+ Err(error) => return Err(error),+ };+ if let Some(jpeg) = jpeg {+ let id = row.frame_id;+ let timestamp = row.timestamp.clone();+ if let Some(fresh) =+ tokio::task::spawn_blocking(move || snapshot_from_jpeg(id, timestamp, &jpeg))+ .await+ .map_err(|e| {+ EnterpriseSyncError::LocalApi(format!("refresh snapshot: {e}"))+ })?+ {+ snapshots.push(fresh);+ }+ }+ }+ *self.snapshots = snapshots;+ Ok(report)+ }+}++/// Re-read only the selected time range, page through timestamp ties, and keep+/// the selected IDs that still exist. Current contents replace stale/redacted+/// copies. New captures are left for the next page instead of joining this one.+async fn filter_current_rows<T, K, F, Fut>(+ selected: &mut Vec<T>,+ limit: u32,+ key: impl Fn(&T) -> K,+ timestamp: impl Fn(&T) -> &str,+ mut fetch: F,+) -> Result<Option<(String, u32)>, EnterpriseSyncError>+where+ K: Eq + std::hash::Hash,+ F: FnMut(String, u32, u32) -> Fut,+ Fut: std::future::Future<Output = Result<Vec<T>, EnterpriseSyncError>>,+{+ let (Some(first), Some(last)) = (selected.first(), selected.last()) else {+ return Ok(None);+ };+ let mut boundary_offset = 0u32;+ let mut rows_at_boundary = 0u32;+ let mut since = Some(timestamp(first).to_owned());+ let last_ts = timestamp(last).to_owned();+ let mut wanted = std::collections::HashMap::new();+ for row in selected.iter() {+ *wanted.entry(key(row)).or_insert(0usize) += 1;+ }+ let selected_count = selected.len();+ let mut matched = 0usize;+ let mut current: std::collections::HashMap<K, std::collections::VecDeque<T>> =+ std::collections::HashMap::new();+ let mut offset = 0;+ loop {+ let page = fetch(since.clone().unwrap(), offset, limit).await?;+ let more = page.len() == limit as usize;+ if page.is_empty() {+ break;+ }+ advance_timestamp_boundary(&mut since, &mut offset, &page, ×tamp);+ for row in page {+ let at_boundary = timestamp(&row) == last_ts;+ if at_boundary {+ rows_at_boundary += 1;+ }+ let id = key(&row);+ if let Some(count) = wanted.get(&id) {+ let values = current.entry(id).or_default();+ if values.len() < *count {+ values.push_back(row);+ matched += 1;+ if at_boundary {+ boundary_offset = rows_at_boundary;+ }+ }+ }+ }+ if !more || since.as_deref().unwrap() > last_ts.as_str() || matched == selected_count {+ break;+ }+ }+ selected.retain_mut(+ |row| match current.get_mut(&key(row)).and_then(|rows| rows.pop_front()) {+ Some(fresh) => {+ *row = fresh;+ true+ }+ None => false,+ },+ );+ // With no surviving selected row at this timestamp, replay from its start.+ // Upload keys are idempotent; replay is preferable to skipping a new row.+ Ok(Some((last_ts, boundary_offset)))+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::sync::{Arc, Mutex};++ fn frame(id: i64, text: &str) -> FrameRow {+ FrameRow {+ frame_id: id,+ timestamp: "2026-10-09T00:00:00Z".into(),+ app_name: None,+ window_name: None,+ browser_url: None,+ text: Some(text.into()),+ accessibility_tree: None,+ }+ }++ #[tokio::test]+ async fn deletion_filters_only_selected_ids_and_recomputes_shifted_timestamp_offset() {+ let mut selected = vec![+ frame(2, "deleted"),+ frame(3, "private"),+ frame(4, "deleted tail"),+ ];+ // Row 1 was acknowledged earlier and has also been deleted. Row 0 is+ // still present before this selection; row 5 is an unselected capture.+ let current = Arc::new(vec![+ frame(0, "earlier"),+ frame(3, "[REDACTED]"),+ frame(5, "new"),+ ]);+ let calls = Arc::new(Mutex::new(Vec::new()));+ let offset = filter_current_rows(&mut selected, 2, |r| r.frame_id, |r| &r.timestamp, {+ let calls = calls.clone();+ move |since, offset, limit| {+ let current = current.clone();+ let calls = calls.clone();+ async move {+ calls.lock().unwrap().push((since, offset));+ Ok(current+ .iter()+ .skip(offset as usize)+ .take(limit as usize)+ .cloned()+ .collect())+ }+ }+ })+ .await+ .unwrap();+ assert_eq!(selected, vec![frame(3, "[REDACTED]")]);+ assert_eq!(+ offset,+ Some(("2026-10-09T00:00:00Z".into(), 2)),+ "count surviving earlier rows as well as selected rows"+ );+ assert_eq!(+ calls+ .lock()+ .unwrap()+ .iter()+ .map(|(_, offset)| *offset)+ .collect::<Vec<_>>(),+ vec![0, 2]+ );+ }++ #[tokio::test]+ async fn completely_deleted_selection_keeps_the_next_capture_for_the_next_page() {+ let mut selected = vec![frame(1, "deleted"), frame(2, "deleted")];+ let offset = filter_current_rows(+ &mut selected,+ 2,+ |r| r.frame_id,+ |r| &r.timestamp,+ |_, _, _| async { Ok(vec![frame(3, "new capture")]) },+ )+ .await+ .unwrap();+ assert!(selected.is_empty());+ assert_eq!(offset, Some(("2026-10-09T00:00:00Z".into(), 0)));+ }++ #[tokio::test]+ async fn source_order_and_repeated_keys_are_preserved() {+ let mut selected = vec![frame(-2, "a"), frame(-2, "b"), frame(-3, "c")];+ let current = Arc::new(vec![+ frame(-1, "earlier"),+ frame(-2, "a redacted"),+ frame(-2, "b redacted"),+ frame(-3, "c redacted"),+ frame(-4, "new"),+ ]);+ let offset = filter_current_rows(+ &mut selected,+ 2,+ |r| r.frame_id,+ |r| &r.timestamp,+ move |_, offset, limit| {+ let current = current.clone();+ async move {+ Ok(current+ .iter()+ .skip(offset as usize)+ .take(limit as usize)+ .cloned()+ .collect())+ }+ },+ )+ .await+ .unwrap();+ assert_eq!(+ selected,+ vec![+ frame(-2, "a redacted"),+ frame(-2, "b redacted"),+ frame(-3, "c redacted")+ ]+ );+ assert_eq!(offset, Some(("2026-10-09T00:00:00Z".into(), 4)));+ }++ #[tokio::test]+ async fn genuine_read_failure_preserves_the_selection_and_original_cause() {+ let mut selected = vec![frame(1, "pending")];+ let before = selected.clone();+ let error = filter_current_rows(+ &mut selected,+ 2,+ |r| r.frame_id,+ |r| &r.timestamp,+ |_, _, _| async {+ Err(EnterpriseSyncError::LocalApi(+ "database disk image is malformed".into(),+ ))+ },+ )+ .await+ .unwrap_err();+ assert!(error+ .to_string()+ .contains("database disk image is malformed"));+ assert_eq!(selected, before);+ }+}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rsindex 562b81492..5e4738067 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs@@ -39,12 +39,9 @@ pub(crate) async fn hydrate_frame_trees(.await.map_err(|e| EnterpriseSyncError::LocalApi(format!("prepare frame AX payloads: {e}")))??;for frame in frames {- frame.accessibility_tree = trees.remove(&frame.frame_id).ok_or_else(|| {- EnterpriseSyncError::LocalApi(format!(- "frame {} disappeared during AX export",- frame.frame_id- ))- })?;+ // Keep the selected ID and scan boundary even if retention removed its+ // payload. The final export filter drops it before serialization.+ frame.accessibility_tree = trees.remove(&frame.frame_id).flatten();}Ok(())}@@ -98,11 +95,15 @@ mod tests {}#[tokio::test]- async fn retained_tree_reaches_wire_and_missing_frames_fail() {+ async fn retained_tree_reaches_wire_and_missing_frames_are_left_for_filtering() {for archived in [false, true] {let dir = tempfile::tempdir().unwrap();let db = if archived {- DatabaseManager::new_hybrid(dir.path(), Default::default(), Default::default())+ // This tiny synthetic fixture tests export contents, not disk+ // admission. Match the DB suite's fixture configuration.+ let mut options = screenpipe_db::storage::MigrationOptions::default();+ options.budget.disk_reserve_bytes = 0;+ DatabaseManager::new_hybrid(dir.path(), Default::default(), options).await.unwrap()} else {@@ -144,9 +145,9 @@ mod tests {"search");assert!(frames[1].accessibility_tree.is_none());- assert!(hydrate_frame_trees(&db, &mut [row(3)], false)- .await- .is_err());+ let mut missing = vec![row(3)];+ hydrate_frame_trees(&db, &mut missing, false).await.unwrap();+ assert!(missing[0].accessibility_tree.is_none());db.close().await;}}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rsindex c352d654f..e89104ca2 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs@@ -19,7 +19,7 @@//!//! # Edge cases handled//!-//! - **Empty batch** — skip POST, advance no cursor, retry next tick+//! - **Empty batch** — skip POST; confirmed deletions still advance the scan boundary//! - **Network failure** — retain the cursor and retry on the normal sync cycle//! - **4xx auth failure** — use the signed-in employee account to fetch the//! rotated device key, then retry without advancing the cursor@@ -313,13 +313,23 @@ impl Cursor {/// inject a mock without spinning up the real server. Implemented in the/// desktop crate against `LocalApiContext`.#[async_trait::async_trait]-pub trait ExportAdmission: Send + Sync {- async fn admit(&self) -> Result<Option<tokio::sync::OwnedMutexGuard<()>>, EnterpriseSyncError>;+pub trait ExportFilter: Send + Sync {+ /// Filter the selected records against their current source state before+ /// serialization. A deletion removes that record, never the whole batch.+ async fn filter(+ &self,+ rows: &mut ExportRows<'_>,+ local: &dyn LocalApiClient,+ ) -> Result<ExportFilterReport, EnterpriseSyncError>;}+#[path = "export_filter.rs"]+mod export_filter;+pub use export_filter::{ExportFilterReport, ExportRows};+#[async_trait::async_trait]pub trait LocalApiClient: Send + Sync {- async fn begin_export(&self) -> Result<Option<Box<dyn ExportAdmission>>, EnterpriseSyncError> {+ async fn begin_export(&self) -> Result<Option<Box<dyn ExportFilter>>, EnterpriseSyncError> {Ok(None)}@@ -576,6 +586,28 @@ pub enum EnterpriseSyncError {Io(#[from] std::io::Error),}+impl EnterpriseSyncError {+ fn is_read_refresh(&self) -> bool {+ let (operation, cause) = match self {+ Self::Storage { operation, cause } => (*operation, cause),+ Self::LocalApi(cause) => ("local read", cause),+ _ => return false,+ };+ let pending = screenpipe_db::storage::is_revision_change_message(cause);+ if pending {+ // Normal support logs explain pending work without reporting an+ // upload error or claiming that this stream was acknowledged.+ info!(+ operation,+ cause,+ retry_seconds = SYNC_INTERVAL.as_secs(),+ "enterprise sync: selection refresh pending; checkpoint retained"+ );+ }+ pending+ }+}+// ─── Pure logic: UTF-8-safe truncation ──────────────────────────────────────/// Truncate `s` to at most `max_bytes`, rounding the cut DOWN to the nearest@@ -864,6 +896,26 @@ async fn run_one_sync_inner(http: &reqwest::Client,include_snapshot: bool,backfill: Option<&backfill::BackfillRequest>,+) -> Result<SyncTickReport, EnterpriseSyncError> {+ match run_one_sync_page(cfg, cursor, local, http, include_snapshot, backfill).await {+ Err(error) if error.is_read_refresh() => {+ debug!("enterprise sync: source changed during read; checkpoint retained for the next cycle");+ Ok(SyncTickReport {+ pending_read: true,+ ..Default::default()+ })+ }+ result => result,+ }+}++async fn run_one_sync_page(+ cfg: &EnterpriseSyncConfig,+ cursor: &mut Cursor,+ local: &dyn LocalApiClient,+ http: &reqwest::Client,+ include_snapshot: bool,+ backfill: Option<&backfill::BackfillRequest>,) -> Result<SyncTickReport, EnterpriseSyncError> {if let EnterpriseUploadMode::Blocked(reason) = &cfg.upload_mode {return Err(EnterpriseSyncError::Configuration(reason.clone()));@@ -967,25 +1019,42 @@ async fn run_one_sync_inner(streams = request.restrict_streams(streams)?;}+ let mut pending_read = false;let mut frames = if streams.frames {- local+ match local.fetch_frames_since(cursor.last_frame_ts.as_deref(),cursor.boundary.frames,FRAME_PAGE_LIMIT,)- .await?+ .await+ {+ Ok(rows) => rows,+ Err(error) if error.is_read_refresh() => {+ pending_read = true;+ Vec::new()+ }+ Err(error) => return Err(error),+ }} else {Vec::new()};let mut audio = if streams.audio {- local+ match local.fetch_audio_since(cursor.last_audio_ts.as_deref(),cursor.boundary.audio,PAGE_LIMIT,)- .await?+ .await+ {+ Ok(rows) => rows,+ Err(error) if error.is_read_refresh() => {+ pending_read = true;+ Vec::new()+ }+ Err(error) => return Err(error),+ }} else {Vec::new()};@@ -998,6 +1067,10 @@ async fn run_one_sync_inner(.await{Ok(rows) => rows,+ Err(e) if e.is_read_refresh() => {+ pending_read = true;+ Vec::new()+ }Err(e) => {if backfill.is_some() {return Err(e);@@ -1013,7 +1086,7 @@ async fn run_one_sync_inner(// not also send the legacy thumbnail stream, even when a cached policy// enables it. Write-only archives cannot use that image channel; retain// one original-quality snapshot in their direct-to-storage JSONL instead.- let snapshots: Vec<SnapshotRow> = if streams.snapshots+ let mut snapshots: Vec<SnapshotRow> = if streams.snapshots&& streams.frame_images == crate::enterprise_policy::FrameImagesMode::All&& include_snapshot&& matches!(cfg.upload_mode, EnterpriseUploadMode::DirectWriteOnly(_))@@ -1021,6 +1094,10 @@ async fn run_one_sync_inner(match local.fetch_latest_snapshot().await {Ok(Some(s)) => vec![s],Ok(None) => Vec::new(),+ Err(e) if e.is_read_refresh() => {+ pending_read = true;+ Vec::new()+ }Err(e) => {warn!("enterprise sync: snapshot fetch failed (skipping): {}", e);Vec::new()@@ -1042,6 +1119,10 @@ async fn run_one_sync_inner(.await{Ok(rows) => rows,+ Err(e) if e.is_read_refresh() => {+ pending_read = true;+ Vec::new()+ }Err(e) => {if backfill.is_some() {return Err(e);@@ -1063,6 +1144,10 @@ async fn run_one_sync_inner(.await{Ok(rows) => rows,+ Err(error) if error.is_read_refresh() => {+ pending_read = true;+ Vec::new()+ }Err(error) => {if backfill.is_some() {return Err(error);@@ -1077,13 +1162,6 @@ async fn run_one_sync_inner(} else {Vec::new()};- if streams.feedback == crate::enterprise_policy::FeedbackSyncMode::Ratings {- for row in &mut feedback {- row.comment = None;- row.snapshot = None;- row.context = serde_json::json!({});- }- }// Parsed app data is a separate privacy-sensitive stream. It is best// effort because parser support is optional and older local servers do// not expose content_type=parsed.@@ -1097,6 +1175,10 @@ async fn run_one_sync_inner(.await{Ok(rows) => rows,+ Err(e) if e.is_read_refresh() => {+ pending_read = true;+ Vec::new()+ }Err(e) => {if backfill.is_some() {return Err(e);@@ -1108,12 +1190,16 @@ async fn run_one_sync_inner(} else {Vec::new()};- let activities = if streams.activities {+ let mut activities = if streams.activities {match local.fetch_activities_since(cursor.boundary.activity_ts.as_deref()).await{Ok(rows) => rows,+ Err(error) if error.is_read_refresh() => {+ pending_read = true;+ Vec::new()+ }Err(error) => {warn!("enterprise sync: activity fetch failed (skipping): {}",@@ -1145,7 +1231,10 @@ async fn run_one_sync_inner(&& feedback.is_empty(){debug!("enterprise sync: nothing new since last tick");- return Ok(SyncTickReport::default());+ return Ok(SyncTickReport {+ pending_read,+ ..Default::default()+ });}// A server restart/database replacement during the reads must not label a@@ -1156,24 +1245,6 @@ async fn run_one_sync_inner());}- let mut body = screenpipe_telemetry_wire::build_jsonl_with_semantic_streams(- &cfg.device_id,- &cfg.device_label,- &frames,- &parsed,- &activities,- &audio,- &ui,- &snapshots,- &memories,- );- body.extend_from_slice(&screenpipe_telemetry_wire::build_feedback_jsonl(- &cfg.device_id,- &cfg.device_label,- &feedback,- ));- let bytes = body.len();-// The history cursor is the retry checkpoint for images as well as text.// Store this page's images first: the existing server sees them as already// resolved, so it need not accumulate another historical request backlog.@@ -1228,25 +1299,123 @@ async fn run_one_sync_inner(|row| &row.timestamp,);- if !backfill.is_some_and(|request| request.images_only) {+ let selected = SyncTickReport {+ frames: frames.len(),+ audio: audio.len(),+ ui: ui.len(),+ memories: memories.len(),+ parsed: parsed.len(),+ feedback: feedback.len(),+ ..Default::default()+ };+ // Keep the original scan boundary when its tail is deleted, and recompute+ // timestamp offsets against the current rows rather than stale positions.+ if let Some(export) = export+ .as_ref()+ .filter(|_| !matches!(cfg.upload_mode, EnterpriseUploadMode::HostedIngest))+ {+ let filtered = export+ .filter(+ &mut ExportRows {+ frames: &mut frames,+ audio: &mut audio,+ ui: &mut ui,+ snapshots: &mut snapshots,+ memories: &mut memories,+ parsed: &mut parsed,+ activities: &mut activities,+ feedback: &mut feedback,+ },+ local,+ )+ .await?;+ filtered.apply(&mut next_cursor);+ }++ if streams.feedback == crate::enterprise_policy::FeedbackSyncMode::Ratings {+ for row in &mut feedback {+ row.comment = None;+ row.snapshot = None;+ row.context = serde_json::json!({});+ }+ }+ // Filtering may read the running local server again. Preserve the source+ // identity check across that extra read boundary as well.+ if cursor.source_id.is_some() && local.upload_source_id().await? != cfg.device_id {+ return Err(EnterpriseSyncError::Configuration(+ "database changed during export filtering; retrying".into(),+ ));+ }++ let mut body = screenpipe_telemetry_wire::build_jsonl_with_semantic_streams(+ &cfg.device_id,+ &cfg.device_label,+ &frames,+ &parsed,+ &activities,+ &audio,+ &ui,+ &snapshots,+ &memories,+ );+ body.extend_from_slice(&screenpipe_telemetry_wire::build_feedback_jsonl(+ &cfg.device_id,+ &cfg.device_label,+ &feedback,+ ));+ let bytes = body.len();++ let mut uploaded = None;+ if !body.is_empty() && !backfill.is_some_and(|request| request.images_only) {match &cfg.upload_mode {EnterpriseUploadMode::HostedIngest => {- for request_body in split_jsonl_requests(body, HOSTED_INGEST_REQUEST_BYTES) {- let admission = match &export {- Some(token) => token.admit().await?,- None => None,- };- let upload = post_jsonl_with_identity(- http,- &cfg.ingest_url,- &cfg.license_key,+ let mut sent = SyncTickReport::default();+ let mut pending = std::collections::VecDeque::from(split_jsonl_requests(+ body,+ HOSTED_INGEST_REQUEST_BYTES,+ ));+ while let Some(request_body) = pending.pop_front() {+ let (request_body, report) = export_filter::filter_request(+ cfg,+ export.as_deref(),+ local,request_body,- cfg.stable_device_id.as_deref(),- backfill.is_some(),- );- drop(admission);- upload.await?;+ &mut next_cursor,+ streams.feedback == crate::enterprise_policy::FeedbackSyncMode::Ratings,+ )+ .await?;+ if cursor.source_id.is_some()+ && local.upload_source_id().await? != cfg.device_id+ {+ return Err(EnterpriseSyncError::Configuration(+ "database changed during export filtering; retrying".into(),+ ));+ }+ // Refreshed contents can grow. Queue the smaller pieces+ // through this same filter so every actual HTTP request+ // has a current selection, including after a network wait.+ let mut pieces =+ split_jsonl_requests(request_body, HOSTED_INGEST_REQUEST_BYTES);+ if pieces.len() > 1 {+ for piece in pieces.into_iter().rev() {+ pending.push_front(piece);+ }+ continue;+ }+ if let Some(request_body) = pieces.pop() {+ post_jsonl_with_identity(+ http,+ &cfg.ingest_url,+ &cfg.license_key,+ request_body,+ cfg.stable_device_id.as_deref(),+ backfill.is_some(),+ )+ .await?;+ }+ sent.add_assign(&report);}+ uploaded = Some(sent);}EnterpriseUploadMode::DirectWriteOnly(direct) => {let counts = DirectUploadRecordCounts {@@ -1259,10 +1428,6 @@ async fn run_one_sync_inner(memories: memories.len(),feedback: feedback.len(),};- let admission = match &export {- Some(token) => token.admit().await?,- None => None,- };let upload = upload_direct_write_only_batch(http,cfg,@@ -1271,7 +1436,6 @@ async fn run_one_sync_inner(counts,enterprise_upload::direct_upload_cursors(&next_cursor),);- drop(admission);upload.await?;}EnterpriseUploadMode::DirectReadable(direct) => {@@ -1285,10 +1449,6 @@ async fn run_one_sync_inner(memories: memories.len(),feedback: feedback.len(),};- let admission = match &export {- Some(token) => token.admit().await?,- None => None,- };let upload = upload_direct_readable_batch(http,cfg,@@ -1298,7 +1458,6 @@ async fn run_one_sync_inner(enterprise_upload::direct_upload_cursors(&next_cursor),backfill.is_some(),);- drop(admission);upload.await?;}EnterpriseUploadMode::Blocked(reason) => {@@ -1306,9 +1465,27 @@ async fn run_one_sync_inner(}}}+ let mut report = uploaded.unwrap_or(SyncTickReport {+ frames: frames.len(),+ parsed: parsed.len(),+ activities: activities.len(),+ audio: audio.len(),+ ui: ui.len(),+ snapshots: snapshots.len(),+ memories: memories.len(),+ feedback: feedback.len(),+ bytes,+ ..Default::default()+ });+ report.scanned_full_page = selected.may_have_more();+ report.pending_read = pending_read;if backfill.is_some() {- let records =- frames.len() + audio.len() + ui.len() + parsed.len() + memories.len() + feedback.len();+ let records = report.frames+ + report.audio+ + report.ui+ + report.parsed+ + report.memories+ + report.feedback;next_cursor.boundary.backfill_records =Some(cursor.boundary.backfill_records.unwrap_or(0) + records as u64);if let Some(images) = images {@@ -1322,17 +1499,7 @@ async fn run_one_sync_inner(*cursor = next_cursor;cursor.save(&cfg.cursor_path)?;- Ok(SyncTickReport {- frames: frames.len(),- parsed: parsed.len(),- activities: activities.len(),- audio: audio.len(),- ui: ui.len(),- snapshots: snapshots.len(),- memories: memories.len(),- feedback: feedback.len(),- bytes,- })+ Ok(report)}fn advance_timestamp_boundary<T>(@@ -1370,6 +1537,10 @@ pub struct SyncTickReport {pub memories: usize,pub feedback: usize,pub bytes: usize,+ /// Filtering must not make a full scanned page look like the end of history.+ pub scanned_full_page: bool,+ /// At least one source read is pending; retain its checkpoint for a later cycle.+ pub pending_read: bool,}impl SyncTickReport {@@ -1377,7 +1548,9 @@ impl SyncTickReport {/// behind it. Point-in-time snapshots are deliberately excluded.pub fn may_have_more(&self) -> bool {let limit = PAGE_LIMIT as usize;- self.frames >= FRAME_PAGE_LIMIT as usize+ self.pending_read+ || self.scanned_full_page+ || self.frames >= FRAME_PAGE_LIMIT as usize|| self.parsed >= limit|| self.audio >= limit|| self.ui >= limit@@ -1388,12 +1561,15 @@ impl SyncTickReport {fn add_assign(&mut self, other: &Self) {self.frames += other.frames;self.parsed += other.parsed;+ self.activities += other.activities;self.audio += other.audio;self.ui += other.ui;self.snapshots += other.snapshots;self.memories += other.memories;self.feedback += other.feedback;self.bytes += other.bytes;+ self.scanned_full_page |= other.scanned_full_page;+ self.pending_read |= other.pending_read;}}@@ -1430,7 +1606,7 @@ pub async fn run_sync_burst(burst.total.add_assign(&page);burst.pages += 1;- if !more_pending {+ if !more_pending || page.pending_read {break;}}@@ -3730,6 +3906,226 @@ pub(crate) mod tests {}}+ #[tokio::test]+ async fn later_upload_requests_filter_deletions_and_redactions_after_an_earlier_ack() {+ let _guard = crate::enterprise_policy::sync_streams_test_lock();+ let _restore = RestoreDefaultSyncStreams;+ crate::enterprise_policy::set_sync_streams(+ true,+ false,+ false,+ false,+ false,+ false,+ "off".into(),+ "off".into(),+ );+ struct CurrentRows(Arc<Mutex<Vec<FrameRow>>>);+ struct CurrentFilter;+ #[async_trait::async_trait]+ impl ExportFilter for CurrentFilter {+ async fn filter(+ &self,+ rows: &mut ExportRows<'_>,+ local: &dyn LocalApiClient,+ ) -> Result<ExportFilterReport, EnterpriseSyncError> {+ rows.refresh(local).await+ }+ }+ #[async_trait::async_trait]+ impl LocalApiClient for CurrentRows {+ async fn begin_export(+ &self,+ ) -> Result<Option<Box<dyn ExportFilter>>, EnterpriseSyncError> {+ Ok(Some(Box::new(CurrentFilter)))+ }+ async fn fetch_frames_since(+ &self,+ since: Option<&str>,+ offset: u32,+ limit: u32,+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {+ Ok(self+ .0+ .lock()+ .unwrap()+ .iter()+ .filter(|row| since.is_none_or(|since| row.timestamp.as_str() >= since))+ .skip(offset as usize)+ .take(limit as usize)+ .cloned()+ .collect())+ }+ async fn fetch_audio_since(+ &self,+ _: Option<&str>,+ _: u32,+ _: u32,+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {+ Ok(Vec::new())+ }+ }+ let ts = "2026-08-04T00:01:00Z";+ // Two records exceed the actual transport target, so each selected+ // record initially occupies its own HTTP request.+ let text = "x".repeat(HOSTED_INGEST_REQUEST_BYTES / 2);+ let rows = Arc::new(Mutex::new(+ (1..=3)+ .map(|id| frame(id, ts, "Arc", &text))+ .collect::<Vec<_>>(),+ ));+ let server = wiremock::MockServer::start().await;+ let changed = rows.clone();+ let responses = Arc::new(AtomicUsize::new(0));+ wiremock::Mock::given(wiremock::matchers::path("/ingest"))+ .respond_with(move |_: &wiremock::Request| {+ if responses.fetch_add(1, Ordering::SeqCst) == 0 {+ let mut rows = changed.lock().unwrap();+ rows.retain(|row| row.frame_id != 2);+ rows.iter_mut().find(|row| row.frame_id == 3).unwrap().text =+ Some("[REDACTED]".into());+ rows.push(frame(4, ts, "Arc", "new capture"));+ }+ wiremock::ResponseTemplate::new(200)+ })+ .expect(3)+ .mount(&server)+ .await;+ let dir = TempDir::new().unwrap();+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));+ let mut cursor = initialized_cursor();+ let local = CurrentRows(rows);+ let http = reqwest::Client::new();+ let report = run_one_sync(&cfg, &mut cursor, &local, &http)+ .await+ .unwrap();+ assert_eq!(report.frames, 2);+ assert_eq!(cursor.boundary.frames, 2);+ let mut restarted = Cursor::load(&cfg.cursor_path);+ assert_eq!(+ run_one_sync(&cfg, &mut restarted, &local, &http)+ .await+ .unwrap()+ .frames,+ 1+ );+ let requests = server.received_requests().await.unwrap();+ let records: Vec<serde_json::Value> = requests+ .iter()+ .map(|request| serde_json::from_slice(&request.body).unwrap())+ .collect();+ assert_eq!(+ records+ .iter()+ .map(|record| record["frame_id"].as_i64().unwrap())+ .collect::<Vec<_>>(),+ vec![1, 3, 4]+ );+ assert_eq!(records[1]["text"], "[REDACTED]");+ assert_eq!(records[2]["text"], "new capture");+ server.verify().await;+ }++ #[tokio::test]+ async fn changed_read_keeps_only_that_stream_pending_and_recovers_after_restart() {+ let _guard = crate::enterprise_policy::sync_streams_test_lock();+ let _restore = RestoreDefaultSyncStreams;+ crate::enterprise_policy::set_sync_streams(+ true,+ false,+ true,+ false,+ false,+ false,+ "off".into(),+ "off".into(),+ );+ struct ChangingLocal(AtomicUsize);+ #[async_trait::async_trait]+ impl LocalApiClient for ChangingLocal {+ async fn fetch_frames_since(+ &self,+ _: Option<&str>,+ _: u32,+ _: u32,+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {+ if self.0.fetch_add(1, Ordering::SeqCst) == 0 {+ return Err(EnterpriseSyncError::LocalApi(+ "read frame AX payloads: frame storage: read revision changed; retry the complete query".into()));+ }+ Ok(vec![FrameRow {+ frame_id: 1,+ timestamp: "2026-08-04T00:01:00Z".into(),+ app_name: None,+ window_name: None,+ browser_url: None,+ text: Some("retained frame".into()),+ accessibility_tree: None,+ }])+ }+ async fn fetch_audio_since(+ &self,+ _: Option<&str>,+ offset: u32,+ _: u32,+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {+ if offset > 0 {+ return Ok(Vec::new());+ }+ Ok(vec![AudioRow {+ transcription_id: 1,+ timestamp: "2026-08-04T00:01:00Z".into(),+ transcription: "unaffected audio".into(),+ speaker: None,+ device: None,+ }])+ }+ }+ let server = wiremock::MockServer::start().await;+ wiremock::Mock::given(wiremock::matchers::path("/ingest"))+ .respond_with(wiremock::ResponseTemplate::new(200))+ .expect(2)+ .mount(&server)+ .await;+ let dir = TempDir::new().unwrap();+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));+ let mut cursor = initialized_cursor();+ cursor.save(&cfg.cursor_path).unwrap();+ let before_frames = cursor.last_frame_ts.clone();+ let local = ChangingLocal(AtomicUsize::new(0));+ let http = reqwest::Client::new();+ let report = run_sync_burst(&cfg, &mut cursor, &local, &http)+ .await+ .unwrap();+ assert!(report.total.pending_read);+ assert!(+ report.total.may_have_more(),+ "pending history must not be marked complete"+ );+ assert_eq!(report.total.audio, 1);+ assert_eq!(cursor.last_frame_ts, before_frames);+ assert_eq!(+ cursor.last_audio_ts.as_deref(),+ Some("2026-08-04T00:01:00Z")+ );+ assert!(!cfg+ .cursor_path+ .with_file_name(diagnostics::LOG_NAME)+ .exists());+ let mut restarted = Cursor::load(&cfg.cursor_path);+ let report = run_one_sync(&cfg, &mut restarted, &local, &http)+ .await+ .unwrap();+ assert!(!report.pending_read);+ assert_eq!(report.frames, 1);+ assert_eq!(report.audio, 0, "acknowledged audio must not be replayed");+ assert_eq!(+ Cursor::load(&cfg.cursor_path).last_frame_ts.as_deref(),+ Some("2026-08-04T00:01:00Z")+ );+ server.verify().await;+ }+/// Exercise the production worker, including its actual sleeps and/// persisted checkpoint, rather than only checking a delay constant.#[tokio::test(flavor = "current_thread")]@@ -3819,12 +4215,12 @@ pub(crate) mod tests {reply.send(Err(EnterpriseSyncError::Storage {operation: if attempt % 2 == 0 {- "admit export"+ "filter export"} else {"prepare export"},cause: if attempt % 2 == 0 {- "frame storage: read revision changed; retry the complete query"+ "database disk image is malformed"} else {"frame storage: payload read deadline exceeded"}@@ -3908,7 +4304,7 @@ pub(crate) mod tests {let logged = std::fs::read_to_string(log_path).unwrap();assert!(logged.contains("enterprise upload storage prepare export failed"));assert!(logged.contains("frame storage: payload read deadline exceeded"));- assert!(logged.contains("frame storage: read revision changed"));+ assert!(logged.contains("database disk image is malformed"));assert!(logged.contains("pending work retained, retrying in 300s"));assert!(!logged.contains("backing off 3600s"));String::from_utf8(diagnostic.body.clone()).unwrap()@@ -4465,10 +4861,22 @@ pub(crate) mod tests {struct UnavailableTree;#[async_trait::async_trait]impl LocalApiClient for UnavailableTree {- async fn fetch_frames_since(&self, _: Option<&str>, _: u32, _: u32) -> Result<Vec<FrameRow>, EnterpriseSyncError> {- Err(EnterpriseSyncError::LocalApi("read frame AX payloads: unavailable".into()))+ async fn fetch_frames_since(+ &self,+ _: Option<&str>,+ _: u32,+ _: u32,+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {+ Err(EnterpriseSyncError::LocalApi(+ "read frame AX payloads: unavailable".into(),+ ))}- async fn fetch_audio_since(&self, _: Option<&str>, _: u32, _: u32) -> Result<Vec<AudioRow>, EnterpriseSyncError> {+ async fn fetch_audio_since(+ &self,+ _: Option<&str>,+ _: u32,+ _: u32,+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {panic!("failed frame read must abort before other streams")}}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rsindex 0c127d04d..7cac9d0be 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs@@ -193,6 +193,23 @@ mod imp {updated_at: String,}+ async fn local_api_error(response: reqwest::Response, url: &str) -> EnterpriseSyncError {+ let status = response.status();+ let cause = response+ .json::<serde_json::Value>()+ .await+ .ok()+ .and_then(|body| {+ body.get("error")+ .and_then(|v| v.as_str())+ .map(str::to_owned)+ });+ EnterpriseSyncError::LocalApi(match cause {+ Some(cause) => format!("GET {url} -> {status}: {cause}"),+ None => format!("GET {url} -> {status}"),+ })+ }+struct StorageExport {db: Arc<screenpipe_db::DatabaseManager>,token: screenpipe_db::storage::StorageReadToken,@@ -214,17 +231,33 @@ mod imp {}#[async_trait::async_trait]- impl crate::enterprise::sync::ExportAdmission for StorageExport {- async fn admit(+ impl crate::enterprise::sync::ExportFilter for StorageExport {+ async fn filter(&self,- ) -> Result<Option<tokio::sync::OwnedMutexGuard<()>>, EnterpriseSyncError> {- self.token- .admit(&self.db.pool)- .await- .map_err(|error| EnterpriseSyncError::Storage {- operation: "admit export",+ rows: &mut ee_sync::ExportRows<'_>,+ local: &dyn LocalApiClient,+ ) -> Result<ee_sync::ExportFilterReport, EnterpriseSyncError> {+ // SQLite has no hybrid revocation counter. Validate its selected+ // rows directly instead of treating a no-op token as freshness.+ if self.db.storage_mode() == screenpipe_db::storage::StorageMode::Sqlite {+ return rows.refresh(local).await;+ }+ match self.token.admit(&self.db.pool).await {+ Ok(guard) => {+ drop(guard);+ Ok(Default::default())+ }+ Err(error) if screenpipe_db::storage::is_revision_change(&error) => {+ // A changed revision is a request to filter the selected+ // rows, not an upload failure. Reads return the current+ // contents of survivors, including applied redactions.+ rows.refresh(local).await+ }+ Err(error) => Err(EnterpriseSyncError::Storage {+ operation: "filter export",cause: error.to_string(),- })+ }),+ }}}@@ -232,7 +265,7 @@ mod imp {impl LocalApiClient for ScreenpipeLocalClient {async fn begin_export(&self,- ) -> Result<Option<Box<dyn crate::enterprise::sync::ExportAdmission>>, EnterpriseSyncError>+ ) -> Result<Option<Box<dyn crate::enterprise::sync::ExportFilter>>, EnterpriseSyncError>{let state = self.app.state::<crate::recording::RecordingState>();let db = state@@ -352,11 +385,7 @@ mod imp {.await.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;if !resp.status().is_success() {- return Err(EnterpriseSyncError::LocalApi(format!(- "GET {} -> {}",- url,- resp.status()- )));+ return Err(local_api_error(resp, &url).await);}let body: LocalSearchResponse = resp.json()@@ -424,11 +453,7 @@ mod imp {.await.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;if !resp.status().is_success() {- return Err(EnterpriseSyncError::LocalApi(format!(- "GET {} -> {}",- url,- resp.status()- )));+ return Err(local_api_error(resp, &url).await);}let body: LocalSearchResponse = resp.json()@@ -469,11 +494,7 @@ mod imp {.await.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;if !resp.status().is_success() {- return Err(EnterpriseSyncError::LocalApi(format!(- "GET {} -> {}",- url,- resp.status()- )));+ return Err(local_api_error(resp, &url).await);}let body: LocalSearchResponse = resp.json()@@ -554,11 +575,7 @@ mod imp {.await.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;if !resp.status().is_success() {- return Err(EnterpriseSyncError::LocalApi(format!(- "GET {} -> {}",- url,- resp.status()- )));+ return Err(local_api_error(resp, &url).await);}let body: LocalSearchResponse = resp.json()@@ -629,11 +646,7 @@ mod imp {.await.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;if !resp.status().is_success() {- return Err(EnterpriseSyncError::LocalApi(format!(- "GET {} -> {}",- search_url,- resp.status()- )));+ return Err(local_api_error(resp, &search_url).await);}let body: LocalSearchResponse = resp.json()@@ -738,11 +751,7 @@ mod imp {.await.map_err(|e| EnterpriseSyncError::LocalApi(e.to_string()))?;if !resp.status().is_success() {- return Err(EnterpriseSyncError::LocalApi(format!(- "GET {} -> {}",- url,- resp.status()- )));+ return Err(local_api_error(resp, &url).await);}let body: LocalMemoriesResponse = resp.json()@@ -787,11 +796,7 @@ mod imp {.await.map_err(|error| EnterpriseSyncError::LocalApi(error.to_string()))?;if !resp.status().is_success() {- return Err(EnterpriseSyncError::LocalApi(format!(- "GET {} -> {}",- url,- resp.status()- )));+ return Err(local_api_error(resp, &url).await);}let body: LocalFeedbackResponse = resp.json()@@ -2317,19 +2322,35 @@ mod imp {}#[tokio::test]- async fn export_upload_proceeds_during_capture_and_rejects_deleted_data() {+ async fn export_local_read_change_preserves_the_source_cause() {+ let server = wiremock::MockServer::start().await;+ wiremock::Mock::given(wiremock::matchers::path("/search"))+ .respond_with(wiremock::ResponseTemplate::new(503).set_body_json(+ serde_json::json!({"error":"frame storage: read revision changed; retry the complete query"})+ )).mount(&server).await;+ let url = format!("{}/search", server.uri());+ let response = reqwest::Client::new().get(&url).send().await.unwrap();+ let error = super::local_api_error(response, &url).await;+ assert!(screenpipe_db::storage::is_revision_change_message(+ &error.to_string()+ ));+ }++ #[tokio::test]+ async fn export_upload_filters_deleted_rows_and_refreshes_redactions() {use super::{AudioRow, EnterpriseSyncError, FrameRow, LocalApiClient, StorageExport};- use crate::enterprise::sync::{self, ExportAdmission};+ use crate::enterprise::sync::{self, ExportFilter};use std::sync::Arc;struct CapturingLocal {db: Arc<screenpipe_db::DatabaseManager>,- delete_selected: bool,+ change: u8,+ selected: std::sync::atomic::AtomicBool,}#[async_trait::async_trait]impl LocalApiClient for CapturingLocal {async fn begin_export(&self,- ) -> Result<Option<Box<dyn ExportAdmission>>, EnterpriseSyncError> {+ ) -> Result<Option<Box<dyn ExportFilter>>, EnterpriseSyncError> {Ok(Some(Box::new(StorageExport::begin(self.db.clone()).await?)))}async fn fetch_frames_since(@@ -2338,30 +2359,52 @@ mod imp {_: u32,_: u32,) -> Result<Vec<FrameRow>, EnterpriseSyncError> {+ let first = !self+ .selected+ .swap(true, std::sync::atomic::Ordering::SeqCst);+ let ids = if first { vec![1, 2] } else { vec![1, 2, 3] };let payloads = self.db- .frame_payloads(&[1], screenpipe_db::storage::Projection::All)+ .frame_payloads(&ids, screenpipe_db::storage::Projection::All).await.unwrap();- let text = payloads.get(&1).unwrap().full_text.clone();- // Capture continues after selecting the outgoing page.- self.db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-10-09','new capture')").await.unwrap();- if self.delete_selected {- self.db- .execute_raw_sql_write("DELETE FROM frames WHERE id=1")- .await- .unwrap();+ let rows = payloads+ .into_iter()+ .map(|(id, payload)| FrameRow {+ accessibility_tree: None,+ frame_id: id,+ timestamp: "2026-10-09T00:00:00Z".into(),+ app_name: None,+ window_name: None,+ browser_url: None,+ text: payload.full_text,+ })+ .collect();+ if first {+ // Recording remains durable while this outgoing page is selected.+ self.db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-10-09','new capture')").await.unwrap();+ match self.change {+ 1 => {+ self.db+ .execute_raw_sql_write("DELETE FROM frames WHERE id=1")+ .await+ .unwrap();+ }+ 2 => {+ self.db.execute_raw_sql_write("UPDATE frames SET full_text='[REDACTED]',full_text_redacted_at=1 WHERE id=1").await.unwrap();+ }+ 3 => {+ self.db+ .execute_raw_sql_write("DELETE FROM frames WHERE id IN (1,2)")+ .await+ .unwrap();+ }+ _ => {}+ }}- Ok(vec![FrameRow {- accessibility_tree: None,- frame_id: 1,- timestamp: "2026-10-09T00:00:00Z".into(),- app_name: None,- window_name: None,- browser_url: None,- text,- }])+ Ok(rows)}+async fn fetch_audio_since(&self,_: Option<&str>,@@ -2371,64 +2414,74 @@ mod imp {Ok(Vec::new())}}- for delete_selected in [false, true] {+ for (change, hybrid) in (0..4).flat_map(|change| [(change, false), (change, true)]) {let dir = tempfile::tempdir().unwrap();- let db = Arc::new(+ let db = Arc::new(if hybrid {+ // Synthetic export fixture; disk admission is covered by+ // the DB tests, independently of developer disk capacity.+ let mut options = screenpipe_db::storage::MigrationOptions::default();+ options.budget.disk_reserve_bytes = 0;screenpipe_db::DatabaseManager::new_hybrid(dir.path(),Default::default(),+ options,+ )+ .await+ .unwrap()+ } else {+ screenpipe_db::DatabaseManager::new(+ dir.path().join("db.sqlite").to_str().unwrap(),Default::default(),).await- .unwrap(),- );- db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-09','pending upload')").await.unwrap();+ .unwrap()+ });+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-09','pending upload'),(2,'2026-10-09','surviving record')").await.unwrap();let server = wiremock::MockServer::start().await;wiremock::Mock::given(wiremock::matchers::path("/ingest")).respond_with(wiremock::ResponseTemplate::new(200))- .expect(if delete_selected { 0 } else { 1 })+ .expect(if change == 3 { 0 } else { 1 }).mount(&server).await;let cfg = sync::tests::test_cfg(&dir, format!("{}/ingest", server.uri()));let mut cursor = sync::Cursor::default();let local = CapturingLocal {db: db.clone(),- delete_selected,+ change,+ selected: std::sync::atomic::AtomicBool::new(false),};let result =sync::run_one_sync(&cfg, &mut cursor, &local, &reqwest::Client::new()).await;- if delete_selected {- let error = result.unwrap_err();- assert!(matches!(- &error,- EnterpriseSyncError::Storage {- operation: "admit export",- ..- }- ));- assert!(error.to_string().contains("read revision changed"));+ let expected = match change {+ 1 => 1,+ 3 => 0,+ _ => 2,+ };+ assert_eq!(result.unwrap().frames, expected);+ assert_eq!(+ sync::Cursor::load(&cfg.cursor_path)+ .last_frame_ts+ .as_deref(),+ Some("2026-10-09T00:00:00Z")+ );+ let requests = server.received_requests().await.unwrap();+ if change != 3 {+ let body = String::from_utf8_lossy(&requests[0].body);+ assert!(body.contains("surviving record"));assert!(- !cfg.cursor_path.exists(),- "revoked data must not be checkpointed"- );- } else {- assert_eq!(result.unwrap().frames, 1);- assert_eq!(- sync::Cursor::load(&cfg.cursor_path)- .last_frame_ts- .as_deref(),- Some("2026-10-09T00:00:00Z")+ !body.contains("new capture"),+ "new rows belong to the next page");- let requests = server.received_requests().await.unwrap();- assert!(String::from_utf8_lossy(&requests[0].body).contains("pending upload"));+ assert_eq!(body.contains("pending upload"), change == 0);+ assert_eq!(body.contains("[REDACTED]"), change == 2);}server.verify().await;let captured = db- .frame_payloads(&[2], screenpipe_db::storage::Projection::All)+ .frame_payloads(&[3], screenpipe_db::storage::Projection::All).await.unwrap();assert_eq!(- captured.get(&2).unwrap().full_text.as_deref(),+ captured.get(&3).unwrap().full_text.as_deref(),Some("new capture"),"upload admission must preserve the new recording");diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rsindex 197eaf0bb..348deca51 100644--- a/crates/screenpipe-db/src/storage/mod.rs+++ b/crates/screenpipe-db/src/storage/mod.rs@@ -37,7 +37,7 @@ pub use lifecycle::{MigrationReport,};pub use maintenance::{compact, export_sqlite, recover_interrupted_compaction};-pub use reader::StorageReadToken;+pub use reader::{is_revision_change, is_revision_change_message, StorageReadToken};use serde::{Deserialize, Serialize};use std::path::{Component, Path, PathBuf};use std::sync::Arc;diff --git a/crates/screenpipe-db/src/storage/reader.rs b/crates/screenpipe-db/src/storage/reader.rsindex db13643c0..c3336b9e1 100644--- a/crates/screenpipe-db/src/storage/reader.rs+++ b/crates/screenpipe-db/src/storage/reader.rs@@ -123,6 +123,9 @@ impl DatabaseManager {/// the duration of a network upload. Final admission still rejects any/// deletion/privacy change committed since this single-statement read.pub async fn storage_export_token(&self) -> Result<StorageReadToken, sqlx::Error> {+ if self.pool.is_closed() {+ return Err(sqlx::Error::PoolClosed);+ }let Some(storage) = &self.storage else {return self.storage_read_token().await;};@@ -287,8 +290,15 @@ impl DatabaseManager {}}-fn is_revision_change(error: &sqlx::Error) -> bool {- matches!(error, sqlx::Error::Protocol(message) if message.starts_with("frame storage: read revision changed;"))+const REVISION_CHANGE_PREFIX: &str = "frame storage: read revision changed;";++/// Recognize invalidated reads after the cause crosses an API/error wrapper.+pub fn is_revision_change_message(message: &str) -> bool {+ message.contains(REVISION_CHANGE_PREFIX)+}++pub fn is_revision_change(error: &sqlx::Error) -> bool {+ matches!(error, sqlx::Error::Protocol(message) if message.starts_with(REVISION_CHANGE_PREFIX))}impl HybridStorage {