Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(enterprise): continue acknowledged backfills without idle delay (#7815)

Ezra Ellette955685bac355
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
index 369f30272..b238075f2 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
@@ -200,7 +200,7 @@ struct Pending {
/// Outcomes acknowledged by storage, checkpointed with the image cursor.
/// Missing pixels are never counted as uploaded screenshots.
-#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub(super) struct ImageProgress {
pub uploaded: u64,
pub unavailable: u64,
@@ -208,7 +208,7 @@ pub(super) struct ImageProgress {
/// Accepted image work remains part of the same request until acknowledged,
/// even when policy revokes pixels or telemetry has reached the end of the range.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub(super) struct ImageReplay {
cursor: Cursor,
pub(super) complete: bool,
@@ -547,6 +547,175 @@ mod tests {
assert!(!policy.audio && !policy.snapshots && !policy.activities);
}
+ #[tokio::test(flavor = "current_thread")]
+ async fn scheduled_image_backfill_continues_before_idle_interval() {
+ let _env = super::super::tests::ENV_LOCK.lock().unwrap();
+ let _guard = crate::enterprise_policy::sync_streams_test_lock();
+ let _restore = super::super::tests::RestoreDefaultSyncStreams;
+ crate::enterprise_policy::set_sync_streams(
+ true,
+ false,
+ false,
+ false,
+ false,
+ false,
+ "off".into(),
+ "all".into(),
+ );
+ struct History {
+ local: Local,
+ jpeg: Vec<u8>,
+ turns: Arc<std::sync::Mutex<Vec<&'static str>>>,
+ }
+ #[async_trait::async_trait]
+ impl LocalApiClient for History {
+ async fn upload_source_id(&self) -> Result<String, EnterpriseSyncError> {
+ self.local.upload_source_id().await
+ }
+ async fn fetch_frames_since(
+ &self,
+ since: Option<&str>,
+ offset: u32,
+ limit: u32,
+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
+ self.turns
+ .lock()
+ .unwrap()
+ .push(if since.unwrap_or("") >= "2026-01-03" {
+ "live"
+ } else {
+ "history"
+ });
+ self.local.fetch_frames_since(since, offset, limit).await
+ }
+ async fn fetch_frame_jpeg(
+ &self,
+ _: i64,
+ ) -> Result<Option<Vec<u8>>, EnterpriseSyncError> {
+ Ok(Some(self.jpeg.clone()))
+ }
+ async fn fetch_audio_since(
+ &self,
+ _: Option<&str>,
+ _: u32,
+ _: u32,
+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
+ panic!("unselected audio");
+ }
+ }
+ let server = MockServer::start().await;
+ Mock::given(method("GET"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .set_body_json(serde_json::json!({"desired_mode": "hosted_ingest"})),
+ )
+ .with_priority(10)
+ .mount(&server)
+ .await;
+ Mock::given(method("GET"))
+ .and(path("/api/enterprise/backfill-requests"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(pending()))
+ .with_priority(1)
+ .mount(&server)
+ .await;
+ let completed = Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let done = completed.clone();
+ Mock::given(method("POST"))
+ .and(path("/api/enterprise/backfill-requests"))
+ .respond_with(move |r: &wiremock::Request| {
+ let body: serde_json::Value = serde_json::from_slice(&r.body).unwrap();
+ if body["status"] == "completed" {
+ assert_eq!(body["uploaded_records"], FRAME_PAGE_LIMIT + 1);
+ assert_eq!(body["screenshots"]["uploaded"], FRAME_PAGE_LIMIT + 1);
+ assert_eq!(body["screenshots"]["unavailable"], 0);
+ done.store(true, Ordering::SeqCst);
+ }
+ ResponseTemplate::new(200).set_body_json(serde_json::json!({"accepted": true}))
+ })
+ .mount(&server)
+ .await;
+ Mock::given(method("POST"))
+ .and(path("/api/enterprise/ingest"))
+ .respond_with(ResponseTemplate::new(200))
+ .expect(2)
+ .mount(&server)
+ .await;
+ Mock::given(method("POST"))
+ .and(path("/api/enterprise/frame-uploads"))
+ .respond_with(|r: &wiremock::Request| {
+ let body: serde_json::Value = serde_json::from_slice(&r.body).unwrap();
+ let ids: Vec<_> = body["frames"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|frame| frame["frame_id"].as_i64().unwrap())
+ .collect();
+ ResponseTemplate::new(200).set_body_json(
+ serde_json::json!({"stored": ids, "unavailable": [], "failed": []}),
+ )
+ })
+ .mount(&server)
+ .await;
+ let dir = tempfile::TempDir::new().unwrap();
+ let cfg = cfg(&dir, &server);
+ let live = Cursor {
+ last_frame_ts: Some("2026-01-03T00:00:00Z".into()),
+ ..Default::default()
+ };
+ live.save(&cfg.cursor_path).unwrap();
+ let mut jpeg = std::io::Cursor::new(Vec::new());
+ image::DynamicImage::new_rgb8(16, 16)
+ .write_to(&mut jpeg, image::ImageFormat::Jpeg)
+ .unwrap();
+ let turns = Arc::new(std::sync::Mutex::new(Vec::new()));
+ let local = Arc::new(History {
+ local: Local {
+ reads: AtomicUsize::new(0),
+ fail_ui: false,
+ },
+ jpeg: jpeg.into_inner(),
+ turns: turns.clone(),
+ });
+ let (_stop, shutdown) = tokio::sync::watch::channel(false);
+ let worker = tokio::spawn(run(cfg.clone(), local, shutdown, None));
+ // Existing worker tests use this ten-second transport deadline. Do not
+ // advance the clock: the regression sleeps for the five-minute idle cycle.
+ let result = tokio::time::timeout(Duration::from_secs(10), async {
+ while !completed.load(Ordering::SeqCst) {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await;
+ worker.abort();
+ assert!(
+ result.is_ok(),
+ "acknowledged historical images must continue without the idle interval"
+ );
+ let turns = turns.lock().unwrap();
+ let live_turns: Vec<_> = turns
+ .iter()
+ .enumerate()
+ .filter_map(|(i, turn)| (*turn == "live").then_some(i))
+ .collect();
+ assert!(
+ live_turns.len() >= 2,
+ "live sync must run between image pages"
+ );
+ assert!(turns[live_turns[0] + 1..live_turns[1]].contains(&"history"));
+ assert!(turns[live_turns[1] + 1..].contains(&"history"));
+ let checkpoint = Cursor::load(&dir.path().join(format!(
+ "enterprise_backfill_{}_{}.json",
+ request().license_id,
+ request().id,
+ )));
+ assert!(checkpoint.boundary.backfill_telemetry_complete);
+ assert!(checkpoint.boundary.backfill_image_replay.unwrap().complete);
+ assert_eq!(
+ Cursor::load(&cfg.cursor_path).last_frame_ts,
+ live.last_frame_ts
+ );
+ }
+
#[tokio::test]
async fn backfill_resumes_timestamp_ties_after_lost_ack_without_touching_live_cursor() {
let _guard = crate::enterprise_policy::sync_streams_test_lock();
@@ -645,6 +814,83 @@ mod tests {
);
}
+ #[tokio::test]
+ async fn pending_backfill_read_keeps_checkpoint_without_immediate_retry() {
+ let _guard = crate::enterprise_policy::sync_streams_test_lock();
+ let _restore = super::super::tests::RestoreDefaultSyncStreams;
+ crate::enterprise_policy::set_sync_streams(
+ true,
+ false,
+ false,
+ false,
+ false,
+ false,
+ "off".into(),
+ "off".into(),
+ );
+ struct PendingRead;
+ #[async_trait::async_trait]
+ impl LocalApiClient for PendingRead {
+ async fn fetch_frames_since(
+ &self,
+ _: Option<&str>,
+ _: u32,
+ _: u32,
+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
+ Err(EnterpriseSyncError::LocalApi(
+ "frame storage: read revision changed; retry pending".into(),
+ ))
+ }
+ async fn fetch_audio_since(
+ &self,
+ _: Option<&str>,
+ _: u32,
+ _: u32,
+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
+ panic!("unselected audio");
+ }
+ }
+ let server = MockServer::start().await;
+ Mock::given(method("GET"))
+ .and(path("/api/enterprise/backfill-requests"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(pending()))
+ .expect(1)
+ .mount(&server)
+ .await;
+ Mock::given(method("POST"))
+ .and(path("/api/enterprise/backfill-requests"))
+ .respond_with(|r: &wiremock::Request| {
+ let body: serde_json::Value = serde_json::from_slice(&r.body).unwrap();
+ assert_eq!(body["status"], "running");
+ assert_eq!(body["uploaded_records"], 0);
+ ResponseTemplate::new(200).set_body_json(serde_json::json!({"accepted": true}))
+ })
+ .expect(2)
+ .mount(&server)
+ .await;
+ let dir = tempfile::TempDir::new().unwrap();
+ let cfg = cfg(&dir, &server);
+ let (_stop, shutdown) = tokio::sync::watch::channel(false);
+ assert!(
+ !fulfill_requests(&cfg, &PendingRead, &enterprise_http_client(), &shutdown)
+ .await
+ .unwrap()
+ );
+ let req = request();
+ let checkpoint = Cursor::load(&dir.path().join(format!(
+ "enterprise_backfill_{}_{}.json",
+ req.license_id, req.id,
+ )));
+ assert_eq!(checkpoint.last_frame_ts, req.initial_cursor().last_frame_ts);
+ assert!(!checkpoint.boundary.backfill_telemetry_complete);
+ assert!(server
+ .received_requests()
+ .await
+ .unwrap()
+ .iter()
+ .all(|r| r.url.path() == "/api/enterprise/backfill-requests"));
+ }
+
#[tokio::test]
async fn backfill_failed_optional_read_does_not_complete_or_upload() {
let _guard = crate::enterprise_policy::sync_streams_test_lock();
@@ -1574,16 +1820,19 @@ mod tests {
}
}
+/// Return whether acknowledged checkpoint progress leaves ready recovery work.
+/// A page yield gives live sync a turn; it must not impose the idle poll delay.
pub(super) async fn fulfill_requests(
cfg: &EnterpriseSyncConfig,
local: &dyn LocalApiClient,
http: &reqwest::Client,
shutdown: &tokio::sync::watch::Receiver<bool>,
-) -> Result<(), EnterpriseSyncError> {
+) -> Result<bool, EnterpriseSyncError> {
let Some(base) = control_plane_base(&cfg.ingest_url) else {
- return Ok(());
+ return Ok(false);
};
let url = format!("{base}/api/enterprise/backfill-requests");
+ let mut more_pending = false;
// At most 10 pages after live sync; poll each page so cancellation and
// server policy changes take effect before the next local read.
for _ in 0..10 {
@@ -1650,6 +1899,7 @@ pub(super) async fn fulfill_requests(
}
};
report(cfg, http, &url, &request, &cursor, "running", None).await?;
+ let before = cursor.clone();
match run_replay_page(&replay_cfg, &mut cursor, local, http, &request).await {
Ok(page) => {
let complete = !page.may_have_more();
@@ -1666,6 +1916,10 @@ pub(super) async fn fulfill_requests(
None,
)
.await?;
+ // Only durable progress can schedule an immediate next turn.
+ // Waiting for a local reader/privacy worker retains the normal
+ // retry cadence rather than repeatedly polling unchanged work.
+ more_pending = !complete && !page.pending_read && cursor != before;
// JPEG pages are heavier than telemetry pages. Yield to live
// sync after one acknowledged image page, retaining its cursor.
if complete
@@ -1682,5 +1936,5 @@ pub(super) async fn fulfill_requests(
}
}
}
- Ok(())
+ Ok(more_pending)
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
index bf0eaa0e4..f64b8adbf 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
@@ -218,7 +218,7 @@ impl EnterpriseSyncConfig {
/// What we persist between runs. RFC3339 string for human-readability + cross-
/// language portability if we ever read it from JS.
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
pub struct Cursor {
#[serde(default)]
pub privacy: std::collections::BTreeMap<String, PrivacyCursor>,
@@ -254,7 +254,7 @@ pub struct Cursor {
boundary: CursorBoundary,
}
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
struct CursorBoundary {
#[serde(default)]
starred_sequence: i64,
@@ -320,7 +320,7 @@ impl Cursor {
}
}
-#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct PrivacyCursor {
pub sequence: i64,
pub cutoff: String,
@@ -3018,6 +3018,7 @@ pub async fn run(
Ok(_) => {}
}
+ let mut continue_backfill = false;
match result {
Ok(report) => {
let total = &report.total;
@@ -3046,10 +3047,9 @@ pub async fn run(
}
// Historical recovery has its own cursor and a bounded work budget.
// Live syncing always runs first. Failure here never rewinds it.
- if let Err(error) =
- backfill::fulfill_requests(&cfg, local.as_ref(), &http, &shutdown).await
- {
- warn!("enterprise backfill: will retry: {}", error);
+ match backfill::fulfill_requests(&cfg, local.as_ref(), &http, &shutdown).await {
+ Ok(more_pending) => continue_backfill = more_pending,
+ Err(error) => warn!("enterprise backfill: will retry: {}", error),
}
// On-demand frame fulfillment — best-effort, gated on the
@@ -3075,6 +3075,15 @@ pub async fn run(
Err(_) => {}
}
+ if continue_backfill {
+ if *shutdown.borrow() {
+ break;
+ }
+ // Start the next turn with live sync and fresh policy/source checks.
+ // Keep page checkpoints, transport sizes and failure retries intact.
+ tokio::task::yield_now().await;
+ continue;
+ }
if sleep_or_shutdown(SYNC_INTERVAL, &mut shutdown).await {
break;
}