YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(enterprise): drain parsed pages at the local query boundary (#7763)
Ezra Ellette223bdcca3ef0
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs b/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rsindex 5e4953e90..66df73ee9 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/diagnostic_logs.rs@@ -262,6 +262,19 @@ async fn build_bundle(files: &[LogFile]) -> String {mod tests {use super::*;+ #[cfg(feature = "enterprise-build")]+ #[tokio::test(flavor = "current_thread")]+ async fn parsed_backlog_failure_and_recovery_survive_collected_support_upload() {+ let logs = tempfile::tempdir().unwrap();+ let report =+ crate::enterprise::sync::tests::check_parsed_backlog_restart(logs.path()).await;+ assert!(report.contains("ingest server error: status 503"));+ assert!(report.contains("pending data retained"));+ assert!(report.contains("pushed 0 frames, 101 parsed"));+ assert!(report.contains("across 2 page(s)"));+ assert!(!report.contains("hunter2"));+ }+#[tokio::test]async fn enterprise_retention_failure_survives_support_redaction() {let root = tempfile::tempdir().unwrap();diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/export_filter.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/export_filter.rsindex e42b22216..1066d93be 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/export_filter.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/export_filter.rs@@ -219,7 +219,7 @@ impl ExportRows<'_> {.await?;report.parsed = filter_current_rows(self.parsed,- PAGE_LIMIT,+ PARSED_PAGE_LIMIT,|r| r.frame_id,|r| r.timestamp.as_str(),|since, offset, limit| async move {diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rsindex 455c69838..704de247c 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs@@ -67,6 +67,9 @@ pub const PAGE_LIMIT: u32 = 500;/// Frames include full AX trees; use smaller pages without truncating nodes.pub const FRAME_PAGE_LIMIT: u32 = 32;+/// Match the parsed local query's existing hydration budget so full pages drain.+pub const PARSED_PAGE_LIMIT: u32 = screenpipe_db::SemanticContextQuery::MAX_PAGE_SIZE;+/// Keep hosted-ingest requests comfortably below Vercel's 4.5 MB function/// body limit. This is a transport envelope, not a data cap: every JSONL line/// is sent, across as many requests as necessary.@@ -1522,7 +1525,7 @@ async fn run_one_sync_page(.fetch_parsed_since(cursor.last_parsed_ts.as_deref(),cursor.boundary.parsed,- PAGE_LIMIT,+ PARSED_PAGE_LIMIT,).await{@@ -1978,7 +1981,7 @@ impl SyncTickReport {|| self.scanned_full_page|| self.frames >= FRAME_PAGE_LIMIT as usize|| self.starred >= limit- || self.parsed >= limit+ || self.parsed >= PARSED_PAGE_LIMIT as usize|| self.audio >= limit|| self.ui >= limit|| self.memories >= limit@@ -2978,6 +2981,7 @@ pub async fn run(Ok(report) => {let total = &report.total;if total.frames > 0+ || total.parsed > 0|| total.audio > 0|| total.ui > 0|| total.snapshots > 0@@ -4090,6 +4094,327 @@ pub(crate) mod tests {// ─── run_one_sync (with mocks) ──────────────────────────────────────+ use screenpipe_db::{DatabaseManager, Order, SemanticContextQuery};++ struct ParsedDatabase {+ db: DatabaseManager,+ refresh: bool,+ }++ struct RefreshParsed;++ #[async_trait::async_trait]+ impl ExportFilter for RefreshParsed {+ async fn admit(&self) -> Result<Option<Box<dyn Send>>, EnterpriseSyncError> {+ Ok(None)+ }++ async fn filter(+ &self,+ rows: &mut ExportRows<'_>,+ local: &dyn LocalApiClient,+ ) -> Result<ExportFilterReport, EnterpriseSyncError> {+ rows.refresh(local).await+ }+ }++ #[async_trait::async_trait]+ impl LocalApiClient for ParsedDatabase {+ async fn begin_export(&self) -> Result<Option<Box<dyn ExportFilter>>, EnterpriseSyncError> {+ Ok(self+ .refresh+ .then(|| Box::new(RefreshParsed) as Box<dyn ExportFilter>))+ }++ async fn fetch_frames_since(+ &self,+ _: Option<&str>,+ _: u32,+ _: u32,+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {+ Ok(Vec::new())+ }++ async fn fetch_audio_since(+ &self,+ _: Option<&str>,+ _: u32,+ _: u32,+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {+ Ok(Vec::new())+ }++ async fn fetch_parsed_since(+ &self,+ since: Option<&str>,+ offset: u32,+ limit: u32,+ ) -> Result<Vec<ParsedRow>, EnterpriseSyncError> {+ // Use the production /search database query and its real page cap.+ let contexts = self+ .db+ .search_semantic_context(&SemanticContextQuery {+ start_time: since.map(|at| at.parse().unwrap()),+ offset,+ limit,+ order: Order::Ascending,+ ..Default::default()+ })+ .await+ .unwrap();+ Ok(contexts+ .into_iter()+ .map(|context| {+ let text = context.render_compact();+ let mut row = serde_json::to_value(context).unwrap();+ row["text"] = text.into();+ serde_json::from_value(row).unwrap()+ })+ .collect())+ }+ }++ async fn parsed_database(root: &std::path::Path, count: u32, refresh: bool) -> ParsedDatabase {+ let db = DatabaseManager::new(+ root.join("parsed.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ db.execute_raw_sql_write(&format!(+ "INSERT INTO semantic_runs(id,input_fingerprint,parser_id,parser_version,schema_version,app_platform,input_content_hash,status,item_count) VALUES(1,zeroblob(32),'test-parser','1',1,'macos',zeroblob(8),'handled',1); \+ INSERT INTO semantic_items(id,entity_fingerprint,version_fingerprint,kind,item_key,identity_quality,body) VALUES(1,zeroblob(32),zeroblob(32),'message','stable-key','stable','synthetic parsed content'); \+ INSERT INTO semantic_run_items(run_id,local_id,item_id,sort_order) VALUES(1,'stable-local',1,0); \+ WITH RECURSIVE n(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM n WHERE id<{count}) \+ INSERT INTO frames(id,timestamp,semantic_run_id) SELECT id,'2026-08-01T00:00:00Z',1 FROM n"+ )).await.unwrap();+ ParsedDatabase { db, refresh }+ }++ fn uploaded_parsed_ids(requests: &[wiremock::Request]) -> Vec<i64> {+ requests+ .iter()+ .filter(|request| request.url.path().ends_with("/ingest"))+ .flat_map(|request| request.body.split(|b| *b == b'\n'))+ .filter(|line| !line.is_empty())+ .map(|line| {+ serde_json::from_slice::<serde_json::Value>(line).unwrap()["frame_id"]+ .as_i64()+ .unwrap()+ })+ .collect()+ }++ #[tokio::test]+ async fn parsed_backlog_drains_real_query_pages_and_refreshes_timestamp_ties() {+ let _guard = crate::enterprise_policy::sync_streams_test_lock();+ let _restore = RestoreDefaultSyncStreams;+ crate::enterprise_policy::set_sync_streams(+ false,+ true,+ false,+ false,+ false,+ false,+ "off".into(),+ "off".into(),+ );+ let http = reqwest::Client::new();+ // Below, at, and above the existing local query boundary, including a+ // later tied page that export refresh must seek past the first page.+ for count in [99, 100, 101, 201] {+ for refresh in [false, true] {+ let dir = TempDir::new().unwrap();+ let local = parsed_database(dir.path(), count, refresh).await;+ let server = wiremock::MockServer::start().await;+ wiremock::Mock::given(wiremock::matchers::method("POST"))+ .respond_with(wiremock::ResponseTemplate::new(200))+ .mount(&server)+ .await;+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));+ let mut cursor = initialized_cursor();+ cursor.last_parsed_ts = Some("2026-07-31T00:00:00Z".into());+ let report = run_sync_burst(&cfg, &mut cursor, &local, &http)+ .await+ .unwrap();+ assert_eq!(+ report.total.parsed, count as usize,+ "count={count}, refresh={refresh}"+ );+ assert_eq!(report.pages, count as usize / 100 + 1);+ let ids = uploaded_parsed_ids(&server.received_requests().await.unwrap());+ assert_eq!(ids, (1..=i64::from(count)).collect::<Vec<_>>());+ let saved = Cursor::load(&cfg.cursor_path);+ assert_eq!(saved.boundary.parsed, count);+ assert_eq!(+ saved.last_parsed_ts.as_deref(),+ Some("2026-08-01T00:00:00Z")+ );+ assert_eq!(+ run_sync_burst(&cfg, &mut cursor, &local, &http)+ .await+ .unwrap()+ .total+ .parsed,+ 0+ );+ local.db.close().await;+ }+ }+ }++ #[tokio::test(flavor = "current_thread")]+ async fn parsed_backlog_failed_page_resumes_from_durable_tie_after_restart() {+ let logs = TempDir::new().unwrap();+ check_parsed_backlog_restart(logs.path()).await;+ }++ // Also run through the real collected/redacted support upload in+ // diagnostic_logs' tests, rather than the isolated target's collector stub.+ pub(crate) async fn check_parsed_backlog_restart(logs: &std::path::Path) -> String {+ let _env = ENV_LOCK.lock().unwrap();+ let _guard = crate::enterprise_policy::sync_streams_test_lock();+ let _restore = RestoreDefaultSyncStreams;+ crate::enterprise_policy::set_sync_streams(+ false,+ true,+ false,+ false,+ false,+ false,+ "off".into(),+ "off".into(),+ );+ let _mode = UploadModeEnvGuard::clear();+ let dir = TempDir::new().unwrap();+ let local = parsed_database(dir.path(), 201, true).await;+ let server = wiremock::MockServer::start().await;+ wiremock::Mock::given(wiremock::matchers::method("GET"))+ .respond_with(+ wiremock::ResponseTemplate::new(200)+ .set_body_json(serde_json::json!({"desired_mode":"hosted_ingest"})),+ )+ .mount(&server)+ .await;+ let calls = Arc::new(AtomicUsize::new(0));+ wiremock::Mock::given(wiremock::matchers::method("POST"))+ .and(wiremock::matchers::path("/api/enterprise/ingest"))+ .respond_with(move |_: &wiremock::Request| {+ wiremock::ResponseTemplate::new(if calls.fetch_add(1, Ordering::SeqCst) == 0 {+ 200+ } else {+ 503+ })+ })+ .expect(2)+ .mount(&server)+ .await;+ let mut cfg = test_cfg(&dir, format!("{}/api/enterprise/ingest", server.uri()));+ cfg.cursor_path = logs.join("cursor.json");+ let mut cursor = initialized_cursor();+ cursor.last_parsed_ts = Some("2026-07-31T00:00:00Z".into());+ let recovery = TestLicenseKeyRecovery::unavailable(Arc::new(AtomicUsize::new(0)));+ let error = run_reported_sync_burst(+ &mut cfg,+ &mut cursor,+ &local,+ &reqwest::Client::new(),+ &recovery,+ )+ .await+ .unwrap_err();+ assert!(matches!(error, EnterpriseSyncError::IngestServerError(503)));+ let saved = Cursor::load(&cfg.cursor_path);+ assert_eq!(saved.boundary.parsed, 100);+ local.db.close().await;+ let reopened = Arc::new(ParsedDatabase {+ db: DatabaseManager::new(+ dir.path().join("parsed.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap(),+ refresh: true,+ });+ let healthy = wiremock::MockServer::start().await;+ wiremock::Mock::given(wiremock::matchers::method("GET"))+ .respond_with(+ wiremock::ResponseTemplate::new(200)+ .set_body_json(serde_json::json!({"desired_mode":"hosted_ingest"})),+ )+ .mount(&healthy)+ .await;+ wiremock::Mock::given(wiremock::matchers::method("POST"))+ .and(wiremock::matchers::path("/api/enterprise/ingest"))+ .respond_with(wiremock::ResponseTemplate::new(200))+ .expect(2)+ .mount(&healthy)+ .await;+ let mut cfg = test_cfg(&dir, format!("{}/api/enterprise/ingest", healthy.uri()));+ cfg.cursor_path = logs.join("cursor.json");+ let log_path = logs.join("screenpipe-app.log");+ let file = std::fs::File::create(&log_path).unwrap();+ let _subscriber = tracing::subscriber::set_default(+ tracing_subscriber::fmt()+ .with_ansi(false)+ .with_writer(move || file.try_clone().unwrap())+ .finish(),+ );+ let (stop, shutdown) = tokio::sync::watch::channel(false);+ let worker = tokio::spawn(run(cfg.clone(), reopened.clone(), shutdown, None));+ tokio::time::timeout(Duration::from_secs(10), async {+ while !std::fs::read_to_string(&log_path)+ .unwrap()+ .contains("pushed 0 frames, 101 parsed")+ {+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("parsed-only recovery must reach normal support logs");+ stop.send(true).unwrap();+ tokio::task::yield_now().await;+ tokio::time::pause();+ tokio::time::advance(SYNC_INTERVAL).await;+ tokio::time::timeout(SYNC_INTERVAL * 2, worker)+ .await+ .unwrap()+ .unwrap();+ tokio::time::resume();+ assert_eq!(+ uploaded_parsed_ids(&healthy.received_requests().await.unwrap()),+ (101..=201).collect::<Vec<_>>()+ );+ assert_eq!(Cursor::load(&cfg.cursor_path).boundary.parsed, 201);+ reopened.db.close().await;+ // Rotation and a later healthy log must preserve the original failure+ // and acknowledged recovery in the actual uploaded report.+ std::fs::rename(&log_path, logs.join("screenpipe-app.2026-10-23.1.log")).unwrap();+ std::fs::write(log_path, "INFO recorder restarted\npassword=hunter2\n").unwrap();+ wiremock::Mock::given(wiremock::matchers::path("/api/logs"))+ .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(+ serde_json::json!({"data":{"signedUrl":format!("{}/support-upload", healthy.uri()),"path":"logs/machine/test/parsed.log"}}),+ )).mount(&healthy).await;+ for path in ["/support-upload", "/api/logs/confirm"] {+ wiremock::Mock::given(wiremock::matchers::path(path))+ .respond_with(wiremock::ResponseTemplate::new(200))+ .expect(1)+ .mount(&healthy)+ .await;+ }+ assert!(+ submit_device_logs(&cfg, &enterprise_http_client(), "parsed backlog regression")+ .await+ .is_some()+ );+ let requests = healthy.received_requests().await.unwrap();+ let upload = requests+ .iter()+ .find(|request| request.url.path() == "/support-upload")+ .unwrap();+ String::from_utf8(upload.body.clone()).unwrap()+ }+struct PrivacyLocal {db: Arc<screenpipe_db::DatabaseManager>,protected: Arc<screenpipe_db::DatabaseManager>,diff --git a/crates/screenpipe-db/src/db/semantic.rs b/crates/screenpipe-db/src/db/semantic.rsindex 35236e33f..91528824d 100644--- a/crates/screenpipe-db/src/db/semantic.rs+++ b/crates/screenpipe-db/src/db/semantic.rs@@ -81,6 +81,12 @@ pub struct SemanticContextQuery {pub order: Order,}+impl SemanticContextQuery {+ /// Existing local-search page budget, applied before hydrating item bodies.+ /// Pagination callers must use this boundary to recognize a full page.+ pub const MAX_PAGE_SIZE: u32 = 100;+}+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, OaSchema)]pub struct SemanticActorAlias {pub id: i64,@@ -561,7 +567,7 @@ impl DatabaseManager {Order::Descending => sql.push("ORDER BY f.timestamp DESC, f.id DESC "),};sql.push("LIMIT ")- .push_bind(query.limit.clamp(1, 100))+ .push_bind(query.limit.clamp(1, SemanticContextQuery::MAX_PAGE_SIZE)).push(" OFFSET ").push_bind(query.offset);