YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(semantic): require known geometry before suppressing off-screen content
Louis Beaumont436be241c547
diff --git a/crates/screenpipe-semantic/evals/context/cases.json b/crates/screenpipe-semantic/evals/context/cases.jsonindex cf220f456..a63b0563b 100644--- a/crates/screenpipe-semantic/evals/context/cases.json+++ b/crates/screenpipe-semantic/evals/context/cases.json@@ -27,7 +27,7 @@{ "role": "AXStaticText", "text": "Alice", "depth": 3, "class_name": "sender" },{ "role": "AXStaticText", "text": "notarization is still blocking", "depth": 3 },{ "role": "AXButton", "text": "Add reaction", "depth": 2 },- { "role": "AXStaticText", "text": "stale hidden Slack instruction", "depth": 1, "on_screen": false }+ { "role": "AXStaticText", "text": "stale hidden Slack instruction", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }]},{@@ -60,7 +60,7 @@{ "role": "AXStaticText", "text": "ChatGPT said:", "depth": 3 },{ "role": "AXStaticText", "text": "Use a bounded replay eval", "depth": 2 },{ "role": "AXButton", "text": "Copy response", "depth": 2 },- { "role": "AXStaticText", "text": "stale hidden ChatGPT answer", "depth": 1, "on_screen": false }+ { "role": "AXStaticText", "text": "stale hidden ChatGPT answer", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }]},{@@ -94,7 +94,7 @@{ "role": "AXGroup", "text": "", "depth": 3, "class_name": "font-claude-response relative" },{ "role": "AXStaticText", "text": "Use exact DOM message boundaries.", "depth": 4 },{ "role": "AXButton", "text": "Copy", "depth": 3 },- { "role": "AXStaticText", "text": "stale hidden Claude answer", "depth": 1, "on_screen": false }+ { "role": "AXStaticText", "text": "stale hidden Claude answer", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }]},{@@ -130,7 +130,7 @@"value": "extract the active note\nreplay archived notes","class_name": "cm-content cm-lineWrapping"},- { "role": "AXTextArea", "text": "stale hidden Obsidian note", "depth": 1, "value": "stale hidden Obsidian note", "on_screen": false }+ { "role": "AXTextArea", "text": "stale hidden Obsidian note", "depth": 1, "value": "stale hidden Obsidian note", "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }]},{@@ -157,7 +157,7 @@{ "role": "Group", "text": "", "depth": 3, "class_name": "message-body" },{ "role": "Text", "text": "Please retry signing.", "depth": 4 },{ "role": "Button", "text": "Archive", "depth": 2 },- { "role": "Text", "text": "stale hidden Gmail message", "depth": 1, "on_screen": false }+ { "role": "Text", "text": "stale hidden Gmail message", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }]},{@@ -185,7 +185,7 @@"value": "Ship semantic context\nMeasure token reduction","help_text": "Launch plan"},- { "role": "AXStaticText", "text": "stale hidden Notion paragraph", "depth": 1, "on_screen": false }+ { "role": "AXStaticText", "text": "stale hidden Notion paragraph", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }]},{@@ -209,7 +209,7 @@{ "role": "Group", "text": "", "depth": 1, "class_name": "task-item completed" },{ "role": "Text", "text": "Define schema", "depth": 2, "class_name": "task-title" },{ "role": "Button", "text": "Add task", "depth": 1 },- { "role": "Text", "text": "stale hidden Todoist task", "depth": 1, "on_screen": false }+ { "role": "Text", "text": "stale hidden Todoist task", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }]},{@@ -243,7 +243,7 @@},{ "role": "AXStaticText", "text": "Parser review", "depth": 2, "class_name": "event-title" },{ "role": "AXButton", "text": "Today", "depth": 1 },- { "role": "AXStaticText", "text": "stale hidden Calendar event", "depth": 1, "on_screen": false }+ { "role": "AXStaticText", "text": "stale hidden Calendar event", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }]},{@@ -266,7 +266,7 @@{ "role": "AXStaticText", "text": "$ cargo test", "depth": 1 },{ "role": "AXStaticText", "text": "24 passed", "depth": 1 },{ "role": "AXButton", "text": "New Tab", "depth": 1 },- { "role": "AXStaticText", "text": "stale hidden terminal command", "depth": 1, "on_screen": false }+ { "role": "AXStaticText", "text": "stale hidden terminal command", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }]},{@@ -301,7 +301,7 @@{ "role": "AXStaticText", "text": "17 passed", "depth": 3 },{ "role": "AXButton", "text": "Explorer", "depth": 2 },{ "role": "AXButton", "text": "Search", "depth": 2 },- { "role": "AXStaticText", "text": "stale hidden editor buffer", "depth": 1, "on_screen": false }+ { "role": "AXStaticText", "text": "stale hidden editor buffer", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }]}]diff --git a/crates/screenpipe-semantic/examples/replay.rs b/crates/screenpipe-semantic/examples/replay.rsindex a688f0284..8ce29b7d7 100644--- a/crates/screenpipe-semantic/examples/replay.rs+++ b/crates/screenpipe-semantic/examples/replay.rs@@ -4,8 +4,8 @@use screenpipe_semantic::{adapt_captured_accessibility_tree, parsers::builtin_parser_registry, render_semantic_context,- AppIdentity, CapturedAccessibilityNode, OutputBudget, ParseContext, ParserRegistry, Platform,- TreeBudget, ValidatedParseOutcome,+ AppIdentity, CapturedAccessibilityNode, NodeBounds, OutputBudget, ParseContext, ParserRegistry,+ Platform, TreeBudget, ValidatedParseOutcome,};use serde::{Deserialize, Serialize};use std::io::{BufRead, Read};@@ -295,6 +295,14 @@ mod tests {role: "AXStaticText".into(),text: "private offscreen sentinel".into(),on_screen: Some(false),+ // Off-window with known geometry: suppression requires that+ // evidence, so keep the sentinel in the suppressed shape.+ bounds: Some(NodeBounds {+ left: 0.1,+ top: -0.9,+ width: 0.6,+ height: 0.15,+ }),..Default::default()},];diff --git a/crates/screenpipe-semantic/src/capture.rs b/crates/screenpipe-semantic/src/capture.rsindex 71d07dd1f..4fc2683e3 100644--- a/crates/screenpipe-semantic/src/capture.rs+++ b/crates/screenpipe-semantic/src/capture.rs@@ -99,6 +99,16 @@ pub struct AdaptedSemanticTree {/// off-screen retain structural fields and flags, but human-readable content is/// withheld so scrollback and hidden overflow cannot become semantic memory./// Unknown visibility remains fail-open.+///+/// Suppression additionally requires retained geometry, because `on_screen`+/// alone does not distinguish the two ways a node can miss the window. A node+/// with bounds that lie off-window is scrollback — exactly what suppression is+/// for. A node with no bounds at all renders no pixels anywhere and tells us+/// nothing about where its content sits; screen-reader affordances+/// (`aria-live` regions, visually-hidden labels) land here and describe what+/// IS on screen, so treating them as scrollback deleted visible content. On+/// real Windows captures that cost ~85% of Claude desktop conversation+/// extraction. No geometry is therefore unknown visibility, and fails open.pub fn adapt_captured_accessibility_tree(nodes: &[CapturedAccessibilityNode],budget: TreeBudget,@@ -152,7 +162,7 @@ pub fn adapt_captured_accessibility_tree(class_count += 1;}let classes = &class_buffer[..class_count];- let known_offscreen = node.on_screen == Some(false);+ let known_offscreen = node.on_screen == Some(false) && node.bounds.is_some();let source_description = node.help_text.as_deref()diff --git a/crates/screenpipe-semantic/src/parsers/families.rs b/crates/screenpipe-semantic/src/parsers/families.rsindex 8ffa0e2a6..f6de600e5 100644--- a/crates/screenpipe-semantic/src/parsers/families.rs+++ b/crates/screenpipe-semantic/src/parsers/families.rs@@ -650,9 +650,14 @@ fn mail_row_cell_text<'a>(tree: &'a SemanticTree, row: NodeId, class: &str) -> Ostruct LabeledTurns {title: String,- messages: Vec<(NodeId, String, String)>,+ /// Same shape the marker path produces: node, actor, actor evidence,+ /// time label, body.+ messages: Vec<(NodeId, String, &'static str, Option<String>, String)>,}+/// Actor evidence for turns identified by their per-turn action controls+/// rather than by direction state or an explicit author label.+const CONTROL_ANCHOR: &str = "control_anchor";const UIA_CONTENT_ROLES: &[&str] = &["Text", "Heading", "Link", "Paragraph", "ListItem"];const MAX_UIA_BUFFER_NODES: usize = 1024;const MAX_UIA_BODY_BYTES: usize = 48 * 1024;@@ -692,7 +697,7 @@ fn labeled_uia_turns(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Optionreturn None;}- let mut messages: Vec<(NodeId, String, String)> = Vec::new();+ let mut messages: Vec<(NodeId, String, &'static str, Option<String>, String)> = Vec::new();let mut buffer: Vec<NodeId> = Vec::new();let mut title: Option<String> = None;let mut last_text: Option<NodeId> = None;@@ -720,7 +725,7 @@ fn labeled_uia_turns(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Optionbuffer.clear();if let Some(body) = collect_text(tree, node) {if messages.len() < MAX_STRUCTURAL_CANDIDATES {- messages.push((node, "[user]".to_owned(), body));+ messages.push((node, "[user]".to_owned(), CONTROL_ANCHOR, None, body));}}skip_under = Some(node);@@ -738,7 +743,13 @@ fn labeled_uia_turns(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Option} else if label.eq_ignore_ascii_case("Good response") {if !buffer.is_empty() && messages.len() < MAX_STRUCTURAL_CANDIDATES {if let Some(body) = joined_buffer_text(tree, &buffer) {- messages.push((buffer[0], profile.display_name.to_owned(), body));+ messages.push((+ buffer[0],+ profile.display_name.to_owned(),+ CONTROL_ANCHOR,+ None,+ body,+ ));}}buffer.clear();@@ -754,7 +765,18 @@ fn labeled_uia_turns(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Option{continue;}- if node_content(tree, node).is_none() {+ let Some(content) = node_content(tree, node) else {+ continue;+ };+ // These surfaces render the turn time as a bare Text sibling right+ // after the turn's own controls. Attach it to the turn just closed+ // and keep it out of the body.+ if content.len() <= 96 && is_message_time_label(content) {+ if let Some(last) = messages.last_mut() {+ last.3.get_or_insert_with(|| {+ content.replace(['\u{00a0}', '\u{202f}'], " ").to_owned()+ });+ }continue;}if role.eq_ignore_ascii_case("Text") {diff --git a/crates/screenpipe-semantic/tests/capture_adapter.rs b/crates/screenpipe-semantic/tests/capture_adapter.rsindex fd6f264b1..2d569f594 100644--- a/crates/screenpipe-semantic/tests/capture_adapter.rs+++ b/crates/screenpipe-semantic/tests/capture_adapter.rs@@ -105,6 +105,14 @@ fn suppresses_known_offscreen_content_without_dropping_structure() {value: Some("hidden value".into()),help_text: Some("hidden description".into()),on_screen: Some(false),+ // Scrolled above the viewport: known geometry, off-window. This is+ // the shape suppression exists for.+ bounds: Some(NodeBounds {+ left: 0.1,+ top: -0.8,+ width: 0.5,+ height: 0.2,+ }),automation_id: Some("message-container".into()),class_name: Some("message assistant".into()),..Default::default()@@ -138,6 +146,32 @@ fn suppresses_known_offscreen_content_without_dropping_structure() {assert_eq!(adapted.stats.value_nodes, 0);}+#[test]+fn keeps_content_for_offscreen_nodes_that_have_no_geometry() {+ // Screen-reader affordances (aria-live announcements, visually-hidden+ // labels) render no pixels anywhere, so the platform reports them as not+ // on screen with no usable rect. They describe content that IS visible,+ // so they are unknown visibility rather than scrollback, and fail open.+ let nodes = vec![CapturedAccessibilityNode {+ role: "AXStaticText".into(),+ text: "You said: ship the parser".into(),+ on_screen: Some(false),+ bounds: None,+ class_name: Some("sr-only select-none".into()),+ ..Default::default()+ }];+ let adapted = adapt_captured_accessibility_tree(&nodes, TreeBudget::default()).unwrap();+ let root = adapted.tree.roots().next().unwrap();++ assert_eq!(adapted.tree.text(root), Some("You said: ship the parser"));+ assert_eq!(adapted.stats.known_offscreen_nodes, 0);+ assert_eq!(adapted.stats.suppressed_offscreen_content_nodes, 0);+ // Structure and flags still record what the platform reported.+ let flags = adapted.tree.flags(root).unwrap();+ assert_ne!(flags & CapturedNodeFlags::ON_SCREEN_KNOWN, 0);+ assert_eq!(flags & CapturedNodeFlags::ON_SCREEN, 0);+}+#[test]fn keeps_content_when_visibility_is_unknown() {let nodes = vec![CapturedAccessibilityNode {