YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix: preserve full AX trees in enterprise frame uploads (#7659)
Louis Beaumont449156e009be
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/data_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/data_sync.rsindex 53d9c86bd..32df537e3 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/data_sync.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/data_sync.rs@@ -297,6 +297,7 @@ mod imp {.into_iter().filter_map(|item| match item {LocalSearchItem::OCR(row) => Some(FrameRow {+ accessibility_tree: None,frame_id: row.frame_id,timestamp: row.timestamp.to_rfc3339(),app_name: nonempty(row.app_name),diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rsindex d3ac541b7..efcbccc68 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs@@ -382,8 +382,9 @@ mod tests {) -> Result<Vec<FrameRow>, EnterpriseSyncError> {self.reads.fetch_add(1, Ordering::SeqCst);let ts = "2026-01-01T01:00:00+00:00";- // 501 rows at one timestamp exercise the durable tie offset.- let rows = (1..=501).map(|id| FrameRow {+ // A full page plus one row at one timestamp exercise the durable tie offset.+ let rows = (1..=super::FRAME_PAGE_LIMIT as i64 + 1).map(|id| FrameRow {+ accessibility_tree: Some(serde_json::json!([{"role":"Button","text":"", "properties":{"automation_id":"search"}}])),frame_id: id,timestamp: ts.into(),app_name: None,@@ -503,12 +504,18 @@ mod tests {.into_iter().filter(|r| r.url.path() == "/api/enterprise/ingest").collect();+ for upload in &uploads {+ for line in String::from_utf8_lossy(&upload.body).lines() {+ let record: serde_json::Value = serde_json::from_str(line).unwrap();+ assert_eq!(record["accessibility_tree"][0]["properties"]["automation_id"], "search");+ }+ }assert_eq!(uploads.iter().map(|r| String::from_utf8_lossy(&r.body).lines().count()).sum::<usize>(),- 501+ super::FRAME_PAGE_LIMIT as usize + 1);assert_eq!(std::fs::read(&cfg.cursor_path).unwrap(),@@ -519,7 +526,7 @@ mod tests {"enterprise_backfill_{}_{}.json",req.license_id, req.id)));- assert_eq!(checkpoint.boundary.backfill_records, Some(501));+ assert_eq!(checkpoint.boundary.backfill_records, Some(super::FRAME_PAGE_LIMIT as u64 + 1));}#[tokio::test]@@ -600,7 +607,7 @@ mod tests {.collect();assert_eq!(manifests[0]["batch_id"], manifests[1]["batch_id"]);assert_eq!(manifests[2]["batch_id"], manifests[3]["batch_id"]);- assert_eq!(manifests[0]["record_counts"]["frames"], 500);+ assert_eq!(manifests[0]["record_counts"]["frames"], super::FRAME_PAGE_LIMIT as u64);}#[tokio::test]@@ -688,7 +695,7 @@ mod tests {run_one_sync_inner(&cfg, &mut cursor, &local, &http, false, Some(&request)).await.unwrap();- assert_eq!(cursor.boundary.backfill_records, Some(500));+ assert_eq!(cursor.boundary.backfill_records, Some(super::FRAME_PAGE_LIMIT as u64));let mut restarted = Cursor::load(&cfg.cursor_path);assert_eq!(restarted@@ -706,13 +713,13 @@ mod tests {.as_ref().unwrap().unavailable,- 500+ super::FRAME_PAGE_LIMIT as u64);run_one_sync_inner(&cfg, &mut restarted, &local, &http, false, Some(&request)).await.unwrap();- assert_eq!(restarted.boundary.backfill_records, Some(501));- assert_eq!(restarted.boundary.backfill_images.unwrap().unavailable, 501);+ assert_eq!(restarted.boundary.backfill_records, Some(super::FRAME_PAGE_LIMIT as u64 + 1));+ assert_eq!(restarted.boundary.backfill_images.unwrap().unavailable, (super::FRAME_PAGE_LIMIT as u64 + 1));let requests = server.received_requests().await.unwrap();assert!(!requests.iter()@@ -768,6 +775,7 @@ mod tests {.skip(offset as usize).take(limit as usize).map(|id| FrameRow {+ accessibility_tree: None,frame_id: id,timestamp: "2026-01-01T01:00:00+00:00".into(),app_name: None,@@ -820,7 +828,7 @@ mod tests {.unwrap();let history = History(jpeg.into_inner());let mut cursor = request.initial_cursor();- for _ in 0..6 {+ for _ in 0..2501_u32.div_ceil(super::FRAME_PAGE_LIMIT) {run_one_sync_inner(&cfg,&mut cursor,@@ -930,7 +938,7 @@ mod tests {));assert_eq!(Cursor::load(&image_checkpoint).boundary.backfill_records,- Some(500)+ Some(super::FRAME_PAGE_LIMIT as u64));fulfill_requests(&cfg, &local, &http, &shutdown).await@@ -941,7 +949,7 @@ mod tests {.backfill_images.unwrap().unavailable,- 501+ (super::FRAME_PAGE_LIMIT as u64 + 1));assert_eq!(std::fs::read_to_string(&cfg.cursor_path).unwrap(),@@ -970,7 +978,7 @@ mod tests {assert_eq!(last["status"], "completed");assert_eq!(last["screenshots"],- serde_json::json!({"uploaded": 0, "unavailable": 501})+ serde_json::json!({"uploaded": 0, "unavailable": (super::FRAME_PAGE_LIMIT as u64 + 1)}));let mut wrong_source = request();wrong_source.images_only = true;@@ -1095,7 +1103,7 @@ mod tests {).await.unwrap();- assert_eq!(page.frames, 500);+ assert_eq!(page.frames, super::FRAME_PAGE_LIMIT as usize);assert_eq!(cursor.source_id.as_deref(), Some(source));assert!(cursor.last_frame_tsdiff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rsnew file mode 100644index 000000000..562b81492--- /dev/null+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs@@ -0,0 +1,153 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++//! Enrich enterprise frame exports from the retained payload, not the lossy+//! search projection. Normal sync and explicit backfill share this path.++use super::{EnterpriseSyncError, FrameRow};+use screenpipe_db::{storage::Projection, DatabaseManager};+use serde_json::Value;++pub(crate) async fn hydrate_frame_trees(+ db: &DatabaseManager,+ frames: &mut [FrameRow],+ redact_pii: bool,+) -> Result<(), EnterpriseSyncError> {+ let ids: Vec<_> = frames.iter().map(|frame| frame.frame_id).collect();+ // This reader handles both SQLite and archived payloads with the existing+ // privacy/revocation checks. The sync export token is admitted again before+ // upload; a failed read must abort the page so its cursor remains retryable.+ let payloads = db+ .frame_payloads(&ids, Projection::Detail)+ .await+ .map_err(|e| EnterpriseSyncError::LocalApi(format!("read frame AX payloads: {e}")))?;+ // Parsing and redacting a full page can be CPU-heavy. Keep it off the+ // async workers that also serve recording and local API requests.+ let mut trees = tokio::task::spawn_blocking(move || {+ payloads+ .into_iter()+ .map(|(id, payload)| {+ let tree = payload+ .accessibility_tree_json+ .as_deref()+ .map(|json| export_tree(json, redact_pii))+ .transpose()?;+ Ok((id, tree))+ })+ .collect::<Result<std::collections::BTreeMap<_, _>, EnterpriseSyncError>>()+ })+ .await+ .map_err(|e| EnterpriseSyncError::LocalApi(format!("prepare frame AX payloads: {e}")))??;+ for frame in frames {+ frame.accessibility_tree = trees.remove(&frame.frame_id).ok_or_else(|| {+ EnterpriseSyncError::LocalApi(format!(+ "frame {} disappeared during AX export",+ frame.frame_id+ ))+ })?;+ }+ Ok(())+}++fn export_tree(json: &str, redact_pii: bool) -> Result<Value, EnterpriseSyncError> {+ let mut tree: Value = serde_json::from_str(json)+ .map_err(|_| EnterpriseSyncError::LocalApi("invalid stored AX tree JSON".into()))?;+ if redact_pii {+ redact_strings(&mut tree);+ }+ Ok(tree)+}++fn redact_strings(value: &mut Value) {+ match value {+ Value::String(text) => *text = screenpipe_core::pii_removal::remove_pii(text),+ Value::Array(items) => items.iter_mut().for_each(redact_strings),+ Value::Object(fields) => fields.values_mut().for_each(redact_strings),+ _ => {}+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn complete_tree_keeps_empty_nodes_and_unknown_properties() {+ let tree = serde_json::json!([+ {"role":"Window","text":"","depth":0},+ {"role":"Button","text":"Search","depth":1,"bounds":[1,2,3,4],+ "properties":{"automation_id":"search","class_name":"Button","future":{"x":true}}}+ ]);+ assert_eq!(export_tree(&tree.to_string(), false).unwrap(), tree);+ }++ #[test]+ fn pii_setting_covers_nested_tree_values_without_removing_structure() {+ let tree = serde_json::json!([{"text":"[email protected]", "properties":{+ "value":"[email protected]", "automation_id":"search", "is_enabled":true},+ "bounds":[10,20,30,40]}]);+ let output = export_tree(&tree.to_string(), true).unwrap();+ assert!(!output.to_string().contains("@example.com"));+ assert_eq!(output[0]["properties"]["automation_id"], "search");+ assert_eq!(output[0]["bounds"], tree[0]["bounds"]);+ }++ #[test]+ fn malformed_tree_fails_instead_of_silently_exporting_text_only() {+ assert!(export_tree("{broken", false).is_err());+ }++ #[tokio::test]+ async fn retained_tree_reaches_wire_and_missing_frames_fail() {+ for archived in [false, true] {+ let dir = tempfile::tempdir().unwrap();+ let db = if archived {+ DatabaseManager::new_hybrid(dir.path(), Default::default(), Default::default())+ .await+ .unwrap()+ } else {+ DatabaseManager::new(+ dir.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap()+ };+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,accessibility_tree_json) VALUES(1,'2026-10-15T00:00:00Z','[{\"role\":\"Button\",\"text\":\"\",\"properties\":{\"automation_id\":\"search\"}}]'),(2,'2026-10-15T00:00:01Z',NULL)").await.unwrap();+ if archived {+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 2);+ }+ let row = |id| FrameRow {+ frame_id: id,+ timestamp: "2026-10-15T00:00:00Z".into(),+ app_name: None,+ window_name: None,+ browser_url: None,+ text: None,+ accessibility_tree: None,+ };+ let mut frames = vec![row(1), row(2)];+ hydrate_frame_trees(&db, &mut frames, false).await.unwrap();+ let body = screenpipe_telemetry_wire::build_jsonl(+ "device",+ "label",+ &frames,+ &[],+ &[],+ &[],+ &[],+ );+ let record: Value =+ serde_json::from_slice(body.split(|b| *b == b'\n').next().unwrap()).unwrap();+ assert_eq!(+ record["accessibility_tree"][0]["properties"]["automation_id"],+ "search"+ );+ assert!(frames[1].accessibility_tree.is_none());+ assert!(hydrate_frame_trees(&db, &mut [row(3)], false)+ .await+ .is_err());+ db.close().await;+ }+ }+}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/source_identity_tests.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/source_identity_tests.rsindex 6e496e2ed..2b40dd6e5 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/source_identity_tests.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/source_identity_tests.rs@@ -72,6 +72,7 @@ impl LocalApiClient for Local {*self.source.lock().unwrap() = RESET.into();}Ok(vec![FrameRow {+ accessibility_tree: None,frame_id: 1,timestamp: chrono::Utc::now().to_rfc3339(),app_name: None,diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rsindex 115eea33e..c352d654f 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs@@ -46,6 +46,8 @@ mod enterprise_upload;mod backfill;#[path = "sync_diagnostics.rs"]mod diagnostics;+#[path = "frame_context.rs"]+pub(crate) mod frame_context;use enterprise_upload::{upload_direct_readable_batch, upload_direct_write_only_batch, DirectUploadRecordCounts,EnterpriseUploadMode,@@ -58,10 +60,13 @@ pub const SYNC_INTERVAL: Duration = Duration::from_secs(5 * 60);/// considered "not enterprise's data" and skipped.pub const SAFE_BACKFILL: Duration = Duration::from_secs(15 * 60);-/// Page size when pulling records from the local API. Keeps each upstream-/// POST under a few hundred KB even on busy machines.+/// Page size for lightweight local API streams. Hosted uploads are also+/// split by serialized bytes below, independently of record count.pub const PAGE_LIMIT: u32 = 500;+/// Frames include full AX trees; use smaller pages without truncating nodes.+pub const FRAME_PAGE_LIMIT: u32 = 32;+/// Keep hosted-ingest requests comfortably below Vercel's 4.5 MB function/// body limit. This is a transport envelope, not a data cap: every JSONL line/// is sent, across as many requests as necessary.@@ -967,7 +972,7 @@ async fn run_one_sync_inner(.fetch_frames_since(cursor.last_frame_ts.as_deref(),cursor.boundary.frames,- PAGE_LIMIT,+ FRAME_PAGE_LIMIT,).await?} else {@@ -1372,7 +1377,7 @@ impl SyncTickReport {/// behind it. Point-in-time snapshots are deliberately excluded.pub fn may_have_more(&self) -> bool {let limit = PAGE_LIMIT as usize;- self.frames >= limit+ self.frames >= FRAME_PAGE_LIMIT as usize|| self.parsed >= limit|| self.audio >= limit|| self.ui >= limit@@ -2772,6 +2777,7 @@ pub(crate) mod tests {fn frame(id: i64, ts: &str, app: &str, text: &str) -> FrameRow {FrameRow {+ accessibility_tree: None,frame_id: id,timestamp: ts.to_string(),app_name: Some(app.to_string()),@@ -4454,6 +4460,31 @@ pub(crate) mod tests {assert_eq!(loaded.last_frame_ts, cursor.last_frame_ts);}+ #[tokio::test]+ async fn ax_read_failure_keeps_cursor_and_uploads_nothing() {+ struct UnavailableTree;+ #[async_trait::async_trait]+ impl LocalApiClient for UnavailableTree {+ async fn fetch_frames_since(&self, _: Option<&str>, _: u32, _: u32) -> Result<Vec<FrameRow>, EnterpriseSyncError> {+ Err(EnterpriseSyncError::LocalApi("read frame AX payloads: unavailable".into()))+ }+ async fn fetch_audio_since(&self, _: Option<&str>, _: u32, _: u32) -> Result<Vec<AudioRow>, EnterpriseSyncError> {+ panic!("failed frame read must abort before other streams")+ }+ }+ let _guard = crate::enterprise_policy::sync_streams_test_lock();+ let server = wiremock::MockServer::start().await;+ let dir = TempDir::new().unwrap();+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));+ let mut cursor = Cursor { last_frame_ts: Some("2026-10-15T00:00:00Z".into()), ..Cursor::default() };+ cursor.save(&cfg.cursor_path).unwrap();+ let before = std::fs::read(&cfg.cursor_path).unwrap();+ assert!(run_one_sync(&cfg, &mut cursor, &UnavailableTree, &reqwest::Client::new()).await.is_err());+ assert_eq!(cursor.last_frame_ts.as_deref(), Some("2026-10-15T00:00:00Z"));+ assert_eq!(std::fs::read(&cfg.cursor_path).unwrap(), before);+ assert!(server.received_requests().await.unwrap().is_empty());+ }+#[tokio::test]async fn old_cursor_drains_every_page_without_waiting_for_normal_ticks() {let server = wiremock::MockServer::start().await;@@ -4464,9 +4495,9 @@ pub(crate) mod tests {.mount(&server).await;- let first = frame_page(1, PAGE_LIMIT as usize);- let second = frame_page(501, PAGE_LIMIT as usize);- let final_page = frame_page(1001, 37);+ let first = frame_page(1, FRAME_PAGE_LIMIT as usize);+ let second = frame_page(FRAME_PAGE_LIMIT as i64 + 1, FRAME_PAGE_LIMIT as usize);+ let final_page = frame_page(2 * FRAME_PAGE_LIMIT as i64 + 1, 17);let expected_last = final_page.last().unwrap().timestamp.clone();let dir = TempDir::new().unwrap();@@ -4489,7 +4520,7 @@ pub(crate) mod tests {.unwrap();assert_eq!(report.pages, 3);- assert_eq!(report.total.frames, 1_037);+ assert_eq!(report.total.frames, 2 * FRAME_PAGE_LIMIT as usize + 17);assert_eq!(cursor.last_frame_ts.as_deref(),Some(expected_last.as_str())@@ -4513,7 +4544,7 @@ pub(crate) mod tests {let tied_at = "2026-08-01T00:00:00Z";let local = InclusiveTimestampLocal {- frames: (1..=PAGE_LIMIT as i64 + 1)+ frames: (1..=FRAME_PAGE_LIMIT as i64 + 1).map(|id| frame(id, tied_at, "Arc", "same timestamp boundary")).collect(),};@@ -4538,10 +4569,10 @@ pub(crate) mod tests {.await.unwrap();- assert_eq!(first.frames, PAGE_LIMIT as usize);+ assert_eq!(first.frames, FRAME_PAGE_LIMIT as usize);assert_eq!(second.frames, 1,- "an inclusive timestamp-only cursor repeats the first 500 tied rows instead of reaching row 501"+ "an inclusive timestamp-only cursor repeats the first full tied page instead of reaching the next row");}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rsindex 3f444a8b0..26ff70075 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs@@ -366,6 +366,7 @@ mod imp {for item in body.data {if let LocalSearchItem::OCR(o) = item {out.push(FrameRow {+ accessibility_tree: None,frame_id: o.frame_id,timestamp: o.timestamp.to_rfc3339(),app_name: Some(o.app_name).filter(|s| !s.is_empty()),@@ -378,6 +379,29 @@ mod imp {// /search returns DESC by default — sort ASC so the cursor advances// monotonically.out.sort_by(|a, b| a.timestamp.cmp(&b.timestamp));+ if !out.is_empty() {+ let state = self.app.state::<crate::recording::RecordingState>();+ let db = state+ .server+ .lock()+ .await+ .as_ref()+ .map(|server| Arc::clone(&server.db))+ .ok_or_else(|| {+ EnterpriseSyncError::LocalApi("recording database is not ready".into())+ })?;+ let settings = crate::store::SettingsStore::get(&self.app)+ .map_err(|e| EnterpriseSyncError::Configuration(e.to_string()))?+ .ok_or_else(|| {+ EnterpriseSyncError::Configuration("recording settings are not ready".into())+ })?;+ ee_sync::frame_context::hydrate_frame_trees(+ &db,+ &mut out,+ settings.recording.use_pii_removal,+ )+ .await?;+ }Ok(out)}@@ -2038,6 +2062,7 @@ mod imp {.unwrap();}Ok(vec![FrameRow {+ accessibility_tree: None,frame_id: 1,timestamp: "2026-10-09T00:00:00Z".into(),app_name: None,diff --git a/crates/screenpipe-gateway/src/api.rs b/crates/screenpipe-gateway/src/api.rsindex 22dabc6e3..55cedafd5 100644--- a/crates/screenpipe-gateway/src/api.rs+++ b/crates/screenpipe-gateway/src/api.rs@@ -2276,6 +2276,7 @@ mod tests {dev,label,&[FrameRow {+ accessibility_tree: None,frame_id: 1,timestamp: format!("2026-07-22T{hour}:00:00Z"),app_name: Some("Arc".to_string()),@@ -3069,6 +3070,7 @@ mod tests {"dev-a","alice-mbp",&[FrameRow {+ accessibility_tree: None,frame_id: 9,timestamp: "2026-07-22T10:00:00Z".to_string(),app_name: None,diff --git a/crates/screenpipe-gateway/src/bin/seed.rs b/crates/screenpipe-gateway/src/bin/seed.rsindex d7d35bd1b..53adbfe9f 100644--- a/crates/screenpipe-gateway/src/bin/seed.rs+++ b/crates/screenpipe-gateway/src/bin/seed.rs@@ -73,6 +73,7 @@ fn batch_for(device: &Device) -> Vec<u8> {device.label,&[FrameRow {+ accessibility_tree: None,frame_id: 1,timestamp: ts(0),app_name: Some("Arc".to_string()),@@ -84,6 +85,7 @@ fn batch_for(device: &Device) -> Vec<u8> {)),},FrameRow {+ accessibility_tree: None,frame_id: 2,timestamp: ts(5),app_name: Some("Slack".to_string()),@@ -158,6 +160,7 @@ fn batch_for(device: &Device) -> Vec<u8> {device.id,device.label,&[FrameRow {+ accessibility_tree: None,frame_id: STALE_FRAME_ID,timestamp: format!("{STALE_DAY}T{:02}:00:00Z", device.hour),app_name: Some("Arc".to_string()),diff --git a/crates/screenpipe-gateway/src/control_plane.rs b/crates/screenpipe-gateway/src/control_plane.rsindex 8826c13c6..83097c642 100644--- a/crates/screenpipe-gateway/src/control_plane.rs+++ b/crates/screenpipe-gateway/src/control_plane.rs@@ -1424,6 +1424,7 @@ mod tests {"dev-a","alice-mbp",&[FrameRow {+ accessibility_tree: None,frame_id: 1,timestamp: "2026-07-22T10:00:00Z".to_string(),app_name: Some("Arc".to_string()),diff --git a/crates/screenpipe-gateway/src/ingest.rs b/crates/screenpipe-gateway/src/ingest.rsindex 71ef8c727..cb1419367 100644--- a/crates/screenpipe-gateway/src/ingest.rs+++ b/crates/screenpipe-gateway/src/ingest.rs@@ -684,7 +684,13 @@ async fn insert_frame(frame: &FrameRow,) -> Result<bool, GatewayError> {let sid = sync_id(device_id, "frame", frame.frame_id);- if frame_row_id_by_sync_id(conn, &sid).await?.is_some() {+ if let Some(id) = frame_row_id_by_sync_id(conn, &sid).await? {+ // An explicit backfill may enrich a frame received from a text-only+ // client. Never let an older replay erase already retained evidence.+ if let Some(tree) = &frame.accessibility_tree {+ sqlx::query("UPDATE frames SET accessibility_tree_json = ?1 WHERE id = ?2 AND accessibility_tree_json IS NULL")+ .bind(tree.to_string()).bind(id).execute(&mut **conn).await?;+ }return Ok(false);}let chunk_id: i64 = sqlx::query_scalar(@@ -700,8 +706,8 @@ async fn insert_frame(sqlx::query(r#"INSERT INTO frames(video_chunk_id, offset_index, timestamp, app_name, window_name, browser_url,- full_text, device_name, sync_id, machine_id, synced_at)- VALUES (?1, 0, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"#,+ full_text, device_name, sync_id, machine_id, synced_at, accessibility_tree_json)+ VALUES (?1, 0, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"#,).bind(chunk_id).bind(&frame.timestamp)@@ -713,6 +719,12 @@ async fn insert_frame(.bind(&sid).bind(device_id).bind(chrono::Utc::now().to_rfc3339())+ .bind(+ frame+ .accessibility_tree+ .as_ref()+ .map(|tree| tree.to_string()),+ ).execute(&mut **conn).await?;Ok(true)@@ -982,6 +994,7 @@ mod tests {device,label,&[FrameRow {+ accessibility_tree: None,frame_id: 1,timestamp: "2026-07-22T10:00:00Z".to_string(),app_name: Some("Arc".to_string()),@@ -1076,6 +1089,54 @@ mod tests {(ingestor, db, src)}+ #[tokio::test]+ async fn full_ax_tree_survives_ingest_backfill_and_legacy_replay() {+ let dir = tempfile::tempdir().unwrap();+ let db = test_db(&dir).await;+ let tree = serde_json::json!([{"role":"Button","text":"","depth":1,+ "properties":{"automation_id":"search","unknown":{"x":true}}}]);+ let mut frame = FrameRow {+ frame_id: 1,+ timestamp: "2026-10-15T00:00:00Z".into(),+ app_name: None,+ window_name: None,+ browser_url: None,+ text: Some("Search".into()),+ accessibility_tree: None,+ };+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ assert!(insert_frame(tx.conn(), "device", "label", &frame)+ .await+ .unwrap());+ frame.accessibility_tree = Some(tree.clone());+ assert!(!insert_frame(tx.conn(), "device", "label", &frame)+ .await+ .unwrap());+ frame.frame_id = 2;+ assert!(insert_frame(tx.conn(), "device", "label", &frame)+ .await+ .unwrap());+ frame.frame_id = 1;+ frame.accessibility_tree = None;+ assert!(!insert_frame(tx.conn(), "device", "label", &frame)+ .await+ .unwrap());+ tx.commit().await.unwrap();+ let rows: Vec<String> =+ sqlx::query_scalar("SELECT accessibility_tree_json FROM frames ORDER BY id")+ .fetch_all(&db.pool)+ .await+ .unwrap();+ assert_eq!(rows.len(), 2);+ for raw in rows {+ assert_eq!(+ serde_json::from_str::<serde_json::Value>(&raw).unwrap(),+ tree+ );+ }+ db.close().await;+ }+#[tokio::test]async fn two_devices_end_up_queryable_in_one_db() {let dir = tempfile::tempdir().unwrap();@@ -1319,6 +1380,7 @@ mod tests {"dev-a","alice-mbp",&[FrameRow {+ accessibility_tree: None,frame_id: 7,timestamp: "2026-07-22T10:00:00Z".to_string(),app_name: Some("Arc".to_string()),@@ -1386,6 +1448,7 @@ mod tests {"dev-a","alice-mbp",&[FrameRow {+ accessibility_tree: None,frame_id: 1,timestamp: "2026-07-22T10:00:00Z".to_string(),app_name: None,@@ -1438,6 +1501,7 @@ mod tests {"dev-a","alice-mbp",&[FrameRow {+ accessibility_tree: None,frame_id: 2,timestamp: "2026-07-22T10:01:00Z".to_string(),app_name: None,@@ -1547,6 +1611,7 @@ mod tests {"dev-a","alice-mbp",&[FrameRow {+ accessibility_tree: None,frame_id: 7,timestamp: "2026-07-22T10:00:00Z".to_string(),app_name: Some("Arc".to_string()),diff --git a/crates/screenpipe-telemetry-wire/src/records.rs b/crates/screenpipe-telemetry-wire/src/records.rsindex 06056330b..f797d5ea1 100644--- a/crates/screenpipe-telemetry-wire/src/records.rs+++ b/crates/screenpipe-telemetry-wire/src/records.rs@@ -30,6 +30,12 @@ pub struct FrameRow {/// Merged accessibility + OCR text (from `frames.full_text`). Already/// PII-sanitized on the device if PII removal is enabled.pub text: Option<String>,+ /// Full captured AX nodes and properties, with device privacy settings+ /// applied. Absent on older clients and frames without retained AX data.+ /// Raw evidence only: consumers should load this on demand, not inline it+ /// into ordinary screen-text search results or agent context.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub accessibility_tree: Option<Value>,}/// One app-specific semantic projection produced from a captured frame.@@ -518,8 +524,26 @@ pub fn parse_jsonl(body: &[u8]) -> ParsedBatch {mod tests {use super::*;+ #[test]+ fn frame_preserves_full_accessibility_tree() {+ let mut record = serde_json::to_value(frame(1, "Search")).unwrap();+ let tree = serde_json::json!([+ {"role":"Window", "text":"", "depth":0, "properties":{"class_name":"Rates"}},+ {"role":"Button", "text":"", "depth":1, "bounds":[10,20,30,40],+ "properties":{"automation_id":"search", "is_enabled":true, "future_property":{"x":1}}}+ ]);+ record["accessibility_tree"] = tree.clone();+ let decoded: FrameRow = serde_json::from_value(record).unwrap();+ let body = build_jsonl("device", "label", &[decoded], &[], &[], &[], &[]);+ let parsed = parse_jsonl(&body);+ assert_eq!(parsed.skipped_lines, 0);+ let round_trip = serde_json::to_value(&parsed.records[0]).unwrap();+ assert_eq!(round_trip["accessibility_tree"], tree);+ }+fn frame(id: i64, text: &str) -> FrameRow {FrameRow {+ accessibility_tree: None,frame_id: id,timestamp: "2026-07-22T10:00:00Z".to_string(),app_name: Some("Arc".to_string()),@@ -622,6 +646,9 @@ mod tests {assert_eq!(line["device_id"], "dev-a");assert_eq!(line["device_label"], "alice-mbp");assert_eq!(line["frame_id"], 42);+ assert!(line.get("accessibility_tree").is_none());+ let legacy: FrameRow = serde_json::from_value(line.clone()).unwrap();+ assert!(legacy.accessibility_tree.is_none());assert!(line.get("frame").is_none(),"row must be flattened, not nested"