Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: preserve full AX trees in enterprise frame uploads (#7659)

Louis Beaumont449156e009be
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/data_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/data_sync.rs
index 53d9c86bd..32df537e3 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/data_sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/data_sync.rs
@@ -297,6 +297,7 @@ mod imp {
.into_iter()
.filter_map(|item| match item {
LocalSearchItem::OCR(row) => Some(FrameRow {
+ accessibility_tree: None,
frame_id: row.frame_id,
timestamp: row.timestamp.to_rfc3339(),
app_name: nonempty(row.app_name),
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
index d3ac541b7..efcbccc68 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/backfill.rs
@@ -382,8 +382,9 @@ mod tests {
) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
self.reads.fetch_add(1, Ordering::SeqCst);
let ts = "2026-01-01T01:00:00+00:00";
- // 501 rows at one timestamp exercise the durable tie offset.
- let rows = (1..=501).map(|id| FrameRow {
+ // A full page plus one row at one timestamp exercise the durable tie offset.
+ let rows = (1..=super::FRAME_PAGE_LIMIT as i64 + 1).map(|id| FrameRow {
+ accessibility_tree: Some(serde_json::json!([{"role":"Button","text":"", "properties":{"automation_id":"search"}}])),
frame_id: id,
timestamp: ts.into(),
app_name: None,
@@ -503,12 +504,18 @@ mod tests {
.into_iter()
.filter(|r| r.url.path() == "/api/enterprise/ingest")
.collect();
+ for upload in &uploads {
+ for line in String::from_utf8_lossy(&upload.body).lines() {
+ let record: serde_json::Value = serde_json::from_str(line).unwrap();
+ assert_eq!(record["accessibility_tree"][0]["properties"]["automation_id"], "search");
+ }
+ }
assert_eq!(
uploads
.iter()
.map(|r| String::from_utf8_lossy(&r.body).lines().count())
.sum::<usize>(),
- 501
+ super::FRAME_PAGE_LIMIT as usize + 1
);
assert_eq!(
std::fs::read(&cfg.cursor_path).unwrap(),
@@ -519,7 +526,7 @@ mod tests {
"enterprise_backfill_{}_{}.json",
req.license_id, req.id
)));
- assert_eq!(checkpoint.boundary.backfill_records, Some(501));
+ assert_eq!(checkpoint.boundary.backfill_records, Some(super::FRAME_PAGE_LIMIT as u64 + 1));
}
#[tokio::test]
@@ -600,7 +607,7 @@ mod tests {
.collect();
assert_eq!(manifests[0]["batch_id"], manifests[1]["batch_id"]);
assert_eq!(manifests[2]["batch_id"], manifests[3]["batch_id"]);
- assert_eq!(manifests[0]["record_counts"]["frames"], 500);
+ assert_eq!(manifests[0]["record_counts"]["frames"], super::FRAME_PAGE_LIMIT as u64);
}
#[tokio::test]
@@ -688,7 +695,7 @@ mod tests {
run_one_sync_inner(&cfg, &mut cursor, &local, &http, false, Some(&request))
.await
.unwrap();
- assert_eq!(cursor.boundary.backfill_records, Some(500));
+ assert_eq!(cursor.boundary.backfill_records, Some(super::FRAME_PAGE_LIMIT as u64));
let mut restarted = Cursor::load(&cfg.cursor_path);
assert_eq!(
restarted
@@ -706,13 +713,13 @@ mod tests {
.as_ref()
.unwrap()
.unavailable,
- 500
+ super::FRAME_PAGE_LIMIT as u64
);
run_one_sync_inner(&cfg, &mut restarted, &local, &http, false, Some(&request))
.await
.unwrap();
- assert_eq!(restarted.boundary.backfill_records, Some(501));
- assert_eq!(restarted.boundary.backfill_images.unwrap().unavailable, 501);
+ assert_eq!(restarted.boundary.backfill_records, Some(super::FRAME_PAGE_LIMIT as u64 + 1));
+ assert_eq!(restarted.boundary.backfill_images.unwrap().unavailable, (super::FRAME_PAGE_LIMIT as u64 + 1));
let requests = server.received_requests().await.unwrap();
assert!(!requests
.iter()
@@ -768,6 +775,7 @@ mod tests {
.skip(offset as usize)
.take(limit as usize)
.map(|id| FrameRow {
+ accessibility_tree: None,
frame_id: id,
timestamp: "2026-01-01T01:00:00+00:00".into(),
app_name: None,
@@ -820,7 +828,7 @@ mod tests {
.unwrap();
let history = History(jpeg.into_inner());
let mut cursor = request.initial_cursor();
- for _ in 0..6 {
+ for _ in 0..2501_u32.div_ceil(super::FRAME_PAGE_LIMIT) {
run_one_sync_inner(
&cfg,
&mut cursor,
@@ -930,7 +938,7 @@ mod tests {
));
assert_eq!(
Cursor::load(&image_checkpoint).boundary.backfill_records,
- Some(500)
+ Some(super::FRAME_PAGE_LIMIT as u64)
);
fulfill_requests(&cfg, &local, &http, &shutdown)
.await
@@ -941,7 +949,7 @@ mod tests {
.backfill_images
.unwrap()
.unavailable,
- 501
+ (super::FRAME_PAGE_LIMIT as u64 + 1)
);
assert_eq!(
std::fs::read_to_string(&cfg.cursor_path).unwrap(),
@@ -970,7 +978,7 @@ mod tests {
assert_eq!(last["status"], "completed");
assert_eq!(
last["screenshots"],
- serde_json::json!({"uploaded": 0, "unavailable": 501})
+ serde_json::json!({"uploaded": 0, "unavailable": (super::FRAME_PAGE_LIMIT as u64 + 1)})
);
let mut wrong_source = request();
wrong_source.images_only = true;
@@ -1095,7 +1103,7 @@ mod tests {
)
.await
.unwrap();
- assert_eq!(page.frames, 500);
+ assert_eq!(page.frames, super::FRAME_PAGE_LIMIT as usize);
assert_eq!(cursor.source_id.as_deref(), Some(source));
assert!(cursor
.last_frame_ts
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs
new file mode 100644
index 000000000..562b81492
--- /dev/null
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/frame_context.rs
@@ -0,0 +1,153 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+//! Enrich enterprise frame exports from the retained payload, not the lossy
+//! search projection. Normal sync and explicit backfill share this path.
+
+use super::{EnterpriseSyncError, FrameRow};
+use screenpipe_db::{storage::Projection, DatabaseManager};
+use serde_json::Value;
+
+pub(crate) async fn hydrate_frame_trees(
+ db: &DatabaseManager,
+ frames: &mut [FrameRow],
+ redact_pii: bool,
+) -> Result<(), EnterpriseSyncError> {
+ let ids: Vec<_> = frames.iter().map(|frame| frame.frame_id).collect();
+ // This reader handles both SQLite and archived payloads with the existing
+ // privacy/revocation checks. The sync export token is admitted again before
+ // upload; a failed read must abort the page so its cursor remains retryable.
+ let payloads = db
+ .frame_payloads(&ids, Projection::Detail)
+ .await
+ .map_err(|e| EnterpriseSyncError::LocalApi(format!("read frame AX payloads: {e}")))?;
+ // Parsing and redacting a full page can be CPU-heavy. Keep it off the
+ // async workers that also serve recording and local API requests.
+ let mut trees = tokio::task::spawn_blocking(move || {
+ payloads
+ .into_iter()
+ .map(|(id, payload)| {
+ let tree = payload
+ .accessibility_tree_json
+ .as_deref()
+ .map(|json| export_tree(json, redact_pii))
+ .transpose()?;
+ Ok((id, tree))
+ })
+ .collect::<Result<std::collections::BTreeMap<_, _>, EnterpriseSyncError>>()
+ })
+ .await
+ .map_err(|e| EnterpriseSyncError::LocalApi(format!("prepare frame AX payloads: {e}")))??;
+ for frame in frames {
+ frame.accessibility_tree = trees.remove(&frame.frame_id).ok_or_else(|| {
+ EnterpriseSyncError::LocalApi(format!(
+ "frame {} disappeared during AX export",
+ frame.frame_id
+ ))
+ })?;
+ }
+ Ok(())
+}
+
+fn export_tree(json: &str, redact_pii: bool) -> Result<Value, EnterpriseSyncError> {
+ let mut tree: Value = serde_json::from_str(json)
+ .map_err(|_| EnterpriseSyncError::LocalApi("invalid stored AX tree JSON".into()))?;
+ if redact_pii {
+ redact_strings(&mut tree);
+ }
+ Ok(tree)
+}
+
+fn redact_strings(value: &mut Value) {
+ match value {
+ Value::String(text) => *text = screenpipe_core::pii_removal::remove_pii(text),
+ Value::Array(items) => items.iter_mut().for_each(redact_strings),
+ Value::Object(fields) => fields.values_mut().for_each(redact_strings),
+ _ => {}
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn complete_tree_keeps_empty_nodes_and_unknown_properties() {
+ let tree = serde_json::json!([
+ {"role":"Window","text":"","depth":0},
+ {"role":"Button","text":"Search","depth":1,"bounds":[1,2,3,4],
+ "properties":{"automation_id":"search","class_name":"Button","future":{"x":true}}}
+ ]);
+ assert_eq!(export_tree(&tree.to_string(), false).unwrap(), tree);
+ }
+
+ #[test]
+ fn pii_setting_covers_nested_tree_values_without_removing_structure() {
+ let tree = serde_json::json!([{"text":"[email protected]", "properties":{
+ "value":"[email protected]", "automation_id":"search", "is_enabled":true},
+ "bounds":[10,20,30,40]}]);
+ let output = export_tree(&tree.to_string(), true).unwrap();
+ assert!(!output.to_string().contains("@example.com"));
+ assert_eq!(output[0]["properties"]["automation_id"], "search");
+ assert_eq!(output[0]["bounds"], tree[0]["bounds"]);
+ }
+
+ #[test]
+ fn malformed_tree_fails_instead_of_silently_exporting_text_only() {
+ assert!(export_tree("{broken", false).is_err());
+ }
+
+ #[tokio::test]
+ async fn retained_tree_reaches_wire_and_missing_frames_fail() {
+ for archived in [false, true] {
+ let dir = tempfile::tempdir().unwrap();
+ let db = if archived {
+ DatabaseManager::new_hybrid(dir.path(), Default::default(), Default::default())
+ .await
+ .unwrap()
+ } else {
+ DatabaseManager::new(
+ dir.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap()
+ };
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,accessibility_tree_json) VALUES(1,'2026-10-15T00:00:00Z','[{\"role\":\"Button\",\"text\":\"\",\"properties\":{\"automation_id\":\"search\"}}]'),(2,'2026-10-15T00:00:01Z',NULL)").await.unwrap();
+ if archived {
+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 2);
+ }
+ let row = |id| FrameRow {
+ frame_id: id,
+ timestamp: "2026-10-15T00:00:00Z".into(),
+ app_name: None,
+ window_name: None,
+ browser_url: None,
+ text: None,
+ accessibility_tree: None,
+ };
+ let mut frames = vec![row(1), row(2)];
+ hydrate_frame_trees(&db, &mut frames, false).await.unwrap();
+ let body = screenpipe_telemetry_wire::build_jsonl(
+ "device",
+ "label",
+ &frames,
+ &[],
+ &[],
+ &[],
+ &[],
+ );
+ let record: Value =
+ serde_json::from_slice(body.split(|b| *b == b'\n').next().unwrap()).unwrap();
+ assert_eq!(
+ record["accessibility_tree"][0]["properties"]["automation_id"],
+ "search"
+ );
+ assert!(frames[1].accessibility_tree.is_none());
+ assert!(hydrate_frame_trees(&db, &mut [row(3)], false)
+ .await
+ .is_err());
+ db.close().await;
+ }
+ }
+}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/source_identity_tests.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/source_identity_tests.rs
index 6e496e2ed..2b40dd6e5 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/source_identity_tests.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/source_identity_tests.rs
@@ -72,6 +72,7 @@ impl LocalApiClient for Local {
*self.source.lock().unwrap() = RESET.into();
}
Ok(vec![FrameRow {
+ accessibility_tree: None,
frame_id: 1,
timestamp: chrono::Utc::now().to_rfc3339(),
app_name: None,
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
index 115eea33e..c352d654f 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
@@ -46,6 +46,8 @@ mod enterprise_upload;
mod backfill;
#[path = "sync_diagnostics.rs"]
mod diagnostics;
+#[path = "frame_context.rs"]
+pub(crate) mod frame_context;
use enterprise_upload::{
upload_direct_readable_batch, upload_direct_write_only_batch, DirectUploadRecordCounts,
EnterpriseUploadMode,
@@ -58,10 +60,13 @@ pub const SYNC_INTERVAL: Duration = Duration::from_secs(5 * 60);
/// considered "not enterprise's data" and skipped.
pub const SAFE_BACKFILL: Duration = Duration::from_secs(15 * 60);
-/// Page size when pulling records from the local API. Keeps each upstream
-/// POST under a few hundred KB even on busy machines.
+/// Page size for lightweight local API streams. Hosted uploads are also
+/// split by serialized bytes below, independently of record count.
pub const PAGE_LIMIT: u32 = 500;
+/// Frames include full AX trees; use smaller pages without truncating nodes.
+pub const FRAME_PAGE_LIMIT: u32 = 32;
+
/// Keep hosted-ingest requests comfortably below Vercel's 4.5 MB function
/// body limit. This is a transport envelope, not a data cap: every JSONL line
/// is sent, across as many requests as necessary.
@@ -967,7 +972,7 @@ async fn run_one_sync_inner(
.fetch_frames_since(
cursor.last_frame_ts.as_deref(),
cursor.boundary.frames,
- PAGE_LIMIT,
+ FRAME_PAGE_LIMIT,
)
.await?
} else {
@@ -1372,7 +1377,7 @@ impl SyncTickReport {
/// behind it. Point-in-time snapshots are deliberately excluded.
pub fn may_have_more(&self) -> bool {
let limit = PAGE_LIMIT as usize;
- self.frames >= limit
+ self.frames >= FRAME_PAGE_LIMIT as usize
|| self.parsed >= limit
|| self.audio >= limit
|| self.ui >= limit
@@ -2772,6 +2777,7 @@ pub(crate) mod tests {
fn frame(id: i64, ts: &str, app: &str, text: &str) -> FrameRow {
FrameRow {
+ accessibility_tree: None,
frame_id: id,
timestamp: ts.to_string(),
app_name: Some(app.to_string()),
@@ -4454,6 +4460,31 @@ pub(crate) mod tests {
assert_eq!(loaded.last_frame_ts, cursor.last_frame_ts);
}
+ #[tokio::test]
+ async fn ax_read_failure_keeps_cursor_and_uploads_nothing() {
+ struct UnavailableTree;
+ #[async_trait::async_trait]
+ impl LocalApiClient for UnavailableTree {
+ async fn fetch_frames_since(&self, _: Option<&str>, _: u32, _: u32) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
+ Err(EnterpriseSyncError::LocalApi("read frame AX payloads: unavailable".into()))
+ }
+ async fn fetch_audio_since(&self, _: Option<&str>, _: u32, _: u32) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
+ panic!("failed frame read must abort before other streams")
+ }
+ }
+ let _guard = crate::enterprise_policy::sync_streams_test_lock();
+ let server = wiremock::MockServer::start().await;
+ let dir = TempDir::new().unwrap();
+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));
+ let mut cursor = Cursor { last_frame_ts: Some("2026-10-15T00:00:00Z".into()), ..Cursor::default() };
+ cursor.save(&cfg.cursor_path).unwrap();
+ let before = std::fs::read(&cfg.cursor_path).unwrap();
+ assert!(run_one_sync(&cfg, &mut cursor, &UnavailableTree, &reqwest::Client::new()).await.is_err());
+ assert_eq!(cursor.last_frame_ts.as_deref(), Some("2026-10-15T00:00:00Z"));
+ assert_eq!(std::fs::read(&cfg.cursor_path).unwrap(), before);
+ assert!(server.received_requests().await.unwrap().is_empty());
+ }
+
#[tokio::test]
async fn old_cursor_drains_every_page_without_waiting_for_normal_ticks() {
let server = wiremock::MockServer::start().await;
@@ -4464,9 +4495,9 @@ pub(crate) mod tests {
.mount(&server)
.await;
- let first = frame_page(1, PAGE_LIMIT as usize);
- let second = frame_page(501, PAGE_LIMIT as usize);
- let final_page = frame_page(1001, 37);
+ let first = frame_page(1, FRAME_PAGE_LIMIT as usize);
+ let second = frame_page(FRAME_PAGE_LIMIT as i64 + 1, FRAME_PAGE_LIMIT as usize);
+ let final_page = frame_page(2 * FRAME_PAGE_LIMIT as i64 + 1, 17);
let expected_last = final_page.last().unwrap().timestamp.clone();
let dir = TempDir::new().unwrap();
@@ -4489,7 +4520,7 @@ pub(crate) mod tests {
.unwrap();
assert_eq!(report.pages, 3);
- assert_eq!(report.total.frames, 1_037);
+ assert_eq!(report.total.frames, 2 * FRAME_PAGE_LIMIT as usize + 17);
assert_eq!(
cursor.last_frame_ts.as_deref(),
Some(expected_last.as_str())
@@ -4513,7 +4544,7 @@ pub(crate) mod tests {
let tied_at = "2026-08-01T00:00:00Z";
let local = InclusiveTimestampLocal {
- frames: (1..=PAGE_LIMIT as i64 + 1)
+ frames: (1..=FRAME_PAGE_LIMIT as i64 + 1)
.map(|id| frame(id, tied_at, "Arc", "same timestamp boundary"))
.collect(),
};
@@ -4538,10 +4569,10 @@ pub(crate) mod tests {
.await
.unwrap();
- assert_eq!(first.frames, PAGE_LIMIT as usize);
+ assert_eq!(first.frames, FRAME_PAGE_LIMIT as usize);
assert_eq!(
second.frames, 1,
- "an inclusive timestamp-only cursor repeats the first 500 tied rows instead of reaching row 501"
+ "an inclusive timestamp-only cursor repeats the first full tied page instead of reaching the next row"
);
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
index 3f444a8b0..26ff70075 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
@@ -366,6 +366,7 @@ mod imp {
for item in body.data {
if let LocalSearchItem::OCR(o) = item {
out.push(FrameRow {
+ accessibility_tree: None,
frame_id: o.frame_id,
timestamp: o.timestamp.to_rfc3339(),
app_name: Some(o.app_name).filter(|s| !s.is_empty()),
@@ -378,6 +379,29 @@ mod imp {
// /search returns DESC by default — sort ASC so the cursor advances
// monotonically.
out.sort_by(|a, b| a.timestamp.cmp(&b.timestamp));
+ if !out.is_empty() {
+ let state = self.app.state::<crate::recording::RecordingState>();
+ let db = state
+ .server
+ .lock()
+ .await
+ .as_ref()
+ .map(|server| Arc::clone(&server.db))
+ .ok_or_else(|| {
+ EnterpriseSyncError::LocalApi("recording database is not ready".into())
+ })?;
+ let settings = crate::store::SettingsStore::get(&self.app)
+ .map_err(|e| EnterpriseSyncError::Configuration(e.to_string()))?
+ .ok_or_else(|| {
+ EnterpriseSyncError::Configuration("recording settings are not ready".into())
+ })?;
+ ee_sync::frame_context::hydrate_frame_trees(
+ &db,
+ &mut out,
+ settings.recording.use_pii_removal,
+ )
+ .await?;
+ }
Ok(out)
}
@@ -2038,6 +2062,7 @@ mod imp {
.unwrap();
}
Ok(vec![FrameRow {
+ accessibility_tree: None,
frame_id: 1,
timestamp: "2026-10-09T00:00:00Z".into(),
app_name: None,
diff --git a/crates/screenpipe-gateway/src/api.rs b/crates/screenpipe-gateway/src/api.rs
index 22dabc6e3..55cedafd5 100644
--- a/crates/screenpipe-gateway/src/api.rs
+++ b/crates/screenpipe-gateway/src/api.rs
@@ -2276,6 +2276,7 @@ mod tests {
dev,
label,
&[FrameRow {
+ accessibility_tree: None,
frame_id: 1,
timestamp: format!("2026-07-22T{hour}:00:00Z"),
app_name: Some("Arc".to_string()),
@@ -3069,6 +3070,7 @@ mod tests {
"dev-a",
"alice-mbp",
&[FrameRow {
+ accessibility_tree: None,
frame_id: 9,
timestamp: "2026-07-22T10:00:00Z".to_string(),
app_name: None,
diff --git a/crates/screenpipe-gateway/src/bin/seed.rs b/crates/screenpipe-gateway/src/bin/seed.rs
index d7d35bd1b..53adbfe9f 100644
--- a/crates/screenpipe-gateway/src/bin/seed.rs
+++ b/crates/screenpipe-gateway/src/bin/seed.rs
@@ -73,6 +73,7 @@ fn batch_for(device: &Device) -> Vec<u8> {
device.label,
&[
FrameRow {
+ accessibility_tree: None,
frame_id: 1,
timestamp: ts(0),
app_name: Some("Arc".to_string()),
@@ -84,6 +85,7 @@ fn batch_for(device: &Device) -> Vec<u8> {
)),
},
FrameRow {
+ accessibility_tree: None,
frame_id: 2,
timestamp: ts(5),
app_name: Some("Slack".to_string()),
@@ -158,6 +160,7 @@ fn batch_for(device: &Device) -> Vec<u8> {
device.id,
device.label,
&[FrameRow {
+ accessibility_tree: None,
frame_id: STALE_FRAME_ID,
timestamp: format!("{STALE_DAY}T{:02}:00:00Z", device.hour),
app_name: Some("Arc".to_string()),
diff --git a/crates/screenpipe-gateway/src/control_plane.rs b/crates/screenpipe-gateway/src/control_plane.rs
index 8826c13c6..83097c642 100644
--- a/crates/screenpipe-gateway/src/control_plane.rs
+++ b/crates/screenpipe-gateway/src/control_plane.rs
@@ -1424,6 +1424,7 @@ mod tests {
"dev-a",
"alice-mbp",
&[FrameRow {
+ accessibility_tree: None,
frame_id: 1,
timestamp: "2026-07-22T10:00:00Z".to_string(),
app_name: Some("Arc".to_string()),
diff --git a/crates/screenpipe-gateway/src/ingest.rs b/crates/screenpipe-gateway/src/ingest.rs
index 71ef8c727..cb1419367 100644
--- a/crates/screenpipe-gateway/src/ingest.rs
+++ b/crates/screenpipe-gateway/src/ingest.rs
@@ -684,7 +684,13 @@ async fn insert_frame(
frame: &FrameRow,
) -> Result<bool, GatewayError> {
let sid = sync_id(device_id, "frame", frame.frame_id);
- if frame_row_id_by_sync_id(conn, &sid).await?.is_some() {
+ if let Some(id) = frame_row_id_by_sync_id(conn, &sid).await? {
+ // An explicit backfill may enrich a frame received from a text-only
+ // client. Never let an older replay erase already retained evidence.
+ if let Some(tree) = &frame.accessibility_tree {
+ sqlx::query("UPDATE frames SET accessibility_tree_json = ?1 WHERE id = ?2 AND accessibility_tree_json IS NULL")
+ .bind(tree.to_string()).bind(id).execute(&mut **conn).await?;
+ }
return Ok(false);
}
let chunk_id: i64 = sqlx::query_scalar(
@@ -700,8 +706,8 @@ async fn insert_frame(
sqlx::query(
r#"INSERT INTO frames
(video_chunk_id, offset_index, timestamp, app_name, window_name, browser_url,
- full_text, device_name, sync_id, machine_id, synced_at)
- VALUES (?1, 0, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"#,
+ full_text, device_name, sync_id, machine_id, synced_at, accessibility_tree_json)
+ VALUES (?1, 0, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"#,
)
.bind(chunk_id)
.bind(&frame.timestamp)
@@ -713,6 +719,12 @@ async fn insert_frame(
.bind(&sid)
.bind(device_id)
.bind(chrono::Utc::now().to_rfc3339())
+ .bind(
+ frame
+ .accessibility_tree
+ .as_ref()
+ .map(|tree| tree.to_string()),
+ )
.execute(&mut **conn)
.await?;
Ok(true)
@@ -982,6 +994,7 @@ mod tests {
device,
label,
&[FrameRow {
+ accessibility_tree: None,
frame_id: 1,
timestamp: "2026-07-22T10:00:00Z".to_string(),
app_name: Some("Arc".to_string()),
@@ -1076,6 +1089,54 @@ mod tests {
(ingestor, db, src)
}
+ #[tokio::test]
+ async fn full_ax_tree_survives_ingest_backfill_and_legacy_replay() {
+ let dir = tempfile::tempdir().unwrap();
+ let db = test_db(&dir).await;
+ let tree = serde_json::json!([{"role":"Button","text":"","depth":1,
+ "properties":{"automation_id":"search","unknown":{"x":true}}}]);
+ let mut frame = FrameRow {
+ frame_id: 1,
+ timestamp: "2026-10-15T00:00:00Z".into(),
+ app_name: None,
+ window_name: None,
+ browser_url: None,
+ text: Some("Search".into()),
+ accessibility_tree: None,
+ };
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ assert!(insert_frame(tx.conn(), "device", "label", &frame)
+ .await
+ .unwrap());
+ frame.accessibility_tree = Some(tree.clone());
+ assert!(!insert_frame(tx.conn(), "device", "label", &frame)
+ .await
+ .unwrap());
+ frame.frame_id = 2;
+ assert!(insert_frame(tx.conn(), "device", "label", &frame)
+ .await
+ .unwrap());
+ frame.frame_id = 1;
+ frame.accessibility_tree = None;
+ assert!(!insert_frame(tx.conn(), "device", "label", &frame)
+ .await
+ .unwrap());
+ tx.commit().await.unwrap();
+ let rows: Vec<String> =
+ sqlx::query_scalar("SELECT accessibility_tree_json FROM frames ORDER BY id")
+ .fetch_all(&db.pool)
+ .await
+ .unwrap();
+ assert_eq!(rows.len(), 2);
+ for raw in rows {
+ assert_eq!(
+ serde_json::from_str::<serde_json::Value>(&raw).unwrap(),
+ tree
+ );
+ }
+ db.close().await;
+ }
+
#[tokio::test]
async fn two_devices_end_up_queryable_in_one_db() {
let dir = tempfile::tempdir().unwrap();
@@ -1319,6 +1380,7 @@ mod tests {
"dev-a",
"alice-mbp",
&[FrameRow {
+ accessibility_tree: None,
frame_id: 7,
timestamp: "2026-07-22T10:00:00Z".to_string(),
app_name: Some("Arc".to_string()),
@@ -1386,6 +1448,7 @@ mod tests {
"dev-a",
"alice-mbp",
&[FrameRow {
+ accessibility_tree: None,
frame_id: 1,
timestamp: "2026-07-22T10:00:00Z".to_string(),
app_name: None,
@@ -1438,6 +1501,7 @@ mod tests {
"dev-a",
"alice-mbp",
&[FrameRow {
+ accessibility_tree: None,
frame_id: 2,
timestamp: "2026-07-22T10:01:00Z".to_string(),
app_name: None,
@@ -1547,6 +1611,7 @@ mod tests {
"dev-a",
"alice-mbp",
&[FrameRow {
+ accessibility_tree: None,
frame_id: 7,
timestamp: "2026-07-22T10:00:00Z".to_string(),
app_name: Some("Arc".to_string()),
diff --git a/crates/screenpipe-telemetry-wire/src/records.rs b/crates/screenpipe-telemetry-wire/src/records.rs
index 06056330b..f797d5ea1 100644
--- a/crates/screenpipe-telemetry-wire/src/records.rs
+++ b/crates/screenpipe-telemetry-wire/src/records.rs
@@ -30,6 +30,12 @@ pub struct FrameRow {
/// Merged accessibility + OCR text (from `frames.full_text`). Already
/// PII-sanitized on the device if PII removal is enabled.
pub text: Option<String>,
+ /// Full captured AX nodes and properties, with device privacy settings
+ /// applied. Absent on older clients and frames without retained AX data.
+ /// Raw evidence only: consumers should load this on demand, not inline it
+ /// into ordinary screen-text search results or agent context.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub accessibility_tree: Option<Value>,
}
/// One app-specific semantic projection produced from a captured frame.
@@ -518,8 +524,26 @@ pub fn parse_jsonl(body: &[u8]) -> ParsedBatch {
mod tests {
use super::*;
+ #[test]
+ fn frame_preserves_full_accessibility_tree() {
+ let mut record = serde_json::to_value(frame(1, "Search")).unwrap();
+ let tree = serde_json::json!([
+ {"role":"Window", "text":"", "depth":0, "properties":{"class_name":"Rates"}},
+ {"role":"Button", "text":"", "depth":1, "bounds":[10,20,30,40],
+ "properties":{"automation_id":"search", "is_enabled":true, "future_property":{"x":1}}}
+ ]);
+ record["accessibility_tree"] = tree.clone();
+ let decoded: FrameRow = serde_json::from_value(record).unwrap();
+ let body = build_jsonl("device", "label", &[decoded], &[], &[], &[], &[]);
+ let parsed = parse_jsonl(&body);
+ assert_eq!(parsed.skipped_lines, 0);
+ let round_trip = serde_json::to_value(&parsed.records[0]).unwrap();
+ assert_eq!(round_trip["accessibility_tree"], tree);
+ }
+
fn frame(id: i64, text: &str) -> FrameRow {
FrameRow {
+ accessibility_tree: None,
frame_id: id,
timestamp: "2026-07-22T10:00:00Z".to_string(),
app_name: Some("Arc".to_string()),
@@ -622,6 +646,9 @@ mod tests {
assert_eq!(line["device_id"], "dev-a");
assert_eq!(line["device_label"], "alice-mbp");
assert_eq!(line["frame_id"], 42);
+ assert!(line.get("accessibility_tree").is_none());
+ let legacy: FrameRow = serde_json::from_value(line.clone()).unwrap();
+ assert!(legacy.accessibility_tree.is_none());
assert!(
line.get("frame").is_none(),
"row must be flattened, not nested"