Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: identify direct-readable historical screenshot uploads (#7427)

Ezra Elletteb256dda3ae51
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
index 4029c1ffd..53526fac6 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
@@ -467,6 +467,109 @@ mod tests {
assert_eq!(manifests[0]["record_counts"]["frames"], 500);
}
+ #[tokio::test]
+ async fn direct_backfill_capacity_failure_drains_and_retries_without_advancing_history() {
+ let _guard = crate::enterprise_policy::sync_streams_test_lock();
+ crate::enterprise_policy::set_sync_streams(
+ true,
+ false,
+ false,
+ false,
+ false,
+ false,
+ "off".into(),
+ "all".into(),
+ );
+ let server = MockServer::start().await;
+ Mock::given(method("POST")).and(path("/ticket"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
+ "ok": true, "method": "PUT", "upload_url": format!("{}/blob", server.uri()), "headers": {}
+ }))).mount(&server).await;
+ Mock::given(method("PUT"))
+ .and(path("/blob"))
+ .respond_with(ResponseTemplate::new(200))
+ .mount(&server)
+ .await;
+ let full = Arc::new(std::sync::atomic::AtomicBool::new(true));
+ let is_full = full.clone();
+ Mock::given(method("POST")).and(path("/complete"))
+ .and(header("x-screenpipe-backfill", "1"))
+ .respond_with(move |_: &wiremock::Request| {
+ if is_full.load(Ordering::SeqCst) {
+ ResponseTemplate::new(429).set_body_json(serde_json::json!({"error":"screenshot queue is full; retry after the device syncs","code":"frame_queue_full"}))
+ } else { ResponseTemplate::new(200) }
+ }).mount(&server).await;
+ Mock::given(method("GET"))
+ .and(path("/api/enterprise/frame-requests"))
+ .respond_with(
+ ResponseTemplate::new(200).set_body_json(serde_json::json!({"frame_ids":[1]})),
+ )
+ .mount(&server)
+ .await;
+ Mock::given(method("POST"))
+ .and(path("/api/enterprise/frame-uploads"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .set_body_json(serde_json::json!({"stored":[1],"failed":[]})),
+ )
+ .mount(&server)
+ .await;
+ let dir = tempfile::TempDir::new().unwrap();
+ let mut cfg = cfg(&dir, &server);
+ cfg.upload_mode =
+ EnterpriseUploadMode::DirectReadable(enterprise_upload::DirectUploadConfig {
+ ticket_url: format!("{}/ticket", server.uri()),
+ complete_url: format!("{}/complete", server.uri()),
+ pinned_hosts: vec![],
+ });
+ let local = Local {
+ reads: AtomicUsize::new(0),
+ fail_ui: false,
+ };
+ let request = request();
+ let mut cursor = request.initial_cursor();
+ let initial = cursor.last_frame_ts.clone();
+ let http = enterprise_http_client();
+ let error = run_one_sync_inner(&cfg, &mut cursor, &local, &http, false, Some(&request))
+ .await
+ .unwrap_err();
+ assert!(error.to_string().contains("frame_queue_full"));
+ assert_eq!(cursor.last_frame_ts, initial);
+ assert_eq!(cursor.boundary.backfill_records, None);
+ assert!(!cfg.cursor_path.exists());
+ // This is the production order after a failed historical pass: screenshot
+ // fulfillment remains reachable, then the same page retries next tick.
+ let drained = fulfill_frame_requests(&cfg, &local, &http).await;
+ assert_eq!(drained.requested, 1);
+ full.store(false, Ordering::SeqCst);
+ run_one_sync_inner(&cfg, &mut cursor, &local, &http, false, Some(&request))
+ .await
+ .unwrap();
+ assert_eq!(cursor.boundary.backfill_records, Some(500));
+ assert_eq!(
+ Cursor::load(&cfg.cursor_path).last_frame_ts,
+ cursor.last_frame_ts
+ );
+ let requests = server.received_requests().await.unwrap();
+ let completions: Vec<serde_json::Value> = requests
+ .iter()
+ .filter(|r| r.url.path() == "/complete")
+ .map(|r| serde_json::from_slice(&r.body).unwrap())
+ .collect();
+ assert!(completions.len() >= 2);
+ assert_eq!(completions.first().unwrap(), completions.last().unwrap());
+ crate::enterprise_policy::set_sync_streams(
+ true,
+ false,
+ true,
+ true,
+ true,
+ true,
+ "off".into(),
+ "cited".into(),
+ );
+ }
+
#[tokio::test]
async fn backfill_wrong_device_and_cancelled_request_never_read_local_data() {
let server = MockServer::start().await;
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
index 49fba1a22..9d9dbb66b 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
@@ -1169,6 +1169,7 @@ async fn run_one_sync_inner(
body,
counts,
enterprise_upload::direct_upload_cursors(&next_cursor),
+ backfill.is_some(),
);
drop(admission);
upload.await?;
@@ -4538,7 +4539,7 @@ pub(crate) mod tests {
async fn direct_upload_completion_auto_cites_only_readable_all_images() {
let _guard = crate::enterprise_policy::sync_streams_test_lock();
let _restore = RestoreDefaultSyncStreams;
- for readable in [false, true] {
+ for (readable, historical) in [(false, false), (true, false), (true, true)] {
for frame_mode in ["off", "cited", "all"] {
let mode_json = format!(
"\"mode\":\"{}\"",
@@ -4624,9 +4625,16 @@ pub(crate) mod tests {
vec![vec![]],
);
let http = reqwest::Client::new();
- let report = run_one_sync(&cfg, &mut cursor, &local, &http)
- .await
- .unwrap();
+ let replay: Option<backfill::BackfillRequest> = historical.then(|| serde_json::from_value(serde_json::json!({
+ "id": "11111111-1111-1111-1111-111111111111",
+ "license_id": "22222222-2222-2222-2222-222222222222",
+ "device_id": cfg.device_id,
+ "start_at": "2026-05-07T09:00:00Z", "end_at": "2026-05-07T11:00:00Z", "streams": ["frames"]
+ })).unwrap());
+ let report =
+ run_one_sync_inner(&cfg, &mut cursor, &local, &http, false, replay.as_ref())
+ .await
+ .unwrap();
assert_eq!(report.frames, 1);
assert_eq!(
@@ -4638,6 +4646,15 @@ pub(crate) mod tests {
.iter()
.find(|r| r.url.path() == "/complete")
.unwrap();
+ assert_eq!(
+ complete.headers.contains_key("x-screenpipe-backfill"),
+ historical
+ );
+ for request in &requests {
+ if request.url.path() == "/blob" {
+ assert!(!request.headers.contains_key("x-screenpipe-backfill"));
+ }
+ }
let body: serde_json::Value = serde_json::from_slice(&complete.body).unwrap();
if readable && frame_mode == "all" {
assert_eq!(body["frame_ids"], serde_json::json!([1]));
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/upload.rs
index 67efad679..952edcdb1 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/upload.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/upload.rs
@@ -436,7 +436,7 @@ pub async fn upload_direct_write_only_batch(
cursors: DirectUploadCursors,
) -> Result<DirectUploadManifest, EnterpriseSyncError> {
let manifest = write_only_direct_upload_manifest(cfg, &plaintext, counts, cursors)?;
- run_ticketed_upload(http, cfg, direct, &manifest, &plaintext).await?;
+ run_ticketed_upload(http, cfg, direct, &manifest, &plaintext, false).await?;
Ok(manifest)
}
@@ -447,9 +447,10 @@ pub async fn upload_direct_readable_batch(
plaintext: Vec<u8>,
counts: DirectUploadRecordCounts,
cursors: DirectUploadCursors,
+ backfill: bool,
) -> Result<DirectUploadManifest, EnterpriseSyncError> {
let manifest = readable_direct_upload_manifest(cfg, &plaintext, counts, cursors)?;
- run_ticketed_upload(http, cfg, direct, &manifest, &plaintext).await?;
+ run_ticketed_upload(http, cfg, direct, &manifest, &plaintext, backfill).await?;
Ok(manifest)
}
@@ -464,6 +465,7 @@ async fn run_ticketed_upload(
direct: &DirectUploadConfig,
manifest: &DirectUploadManifest,
body: &[u8],
+ backfill: bool,
) -> Result<(), EnterpriseSyncError> {
let mut control_headers = HeaderMap::new();
control_headers.insert(
@@ -473,6 +475,12 @@ async fn run_ticketed_upload(
.map_err(|e| EnterpriseSyncError::Ingest(format!("bad license-key header: {e}")))?,
);
+ // Same historical-replay contract as hosted ingest. Keep it on control
+ // requests only; the customer-storage PUT retains its signed headers.
+ if backfill {
+ control_headers.insert("x-screenpipe-backfill", "1".parse().unwrap());
+ }
+
let pipeline_cfg = TicketedConfig::new(direct.ticket_url.clone(), direct.complete_url.clone())
.with_control_headers(control_headers)
.with_http(http.clone())