Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(semantic): require known geometry before suppressing off-screen content

Louis Beaumont436be241c547
diff --git a/crates/screenpipe-semantic/evals/context/cases.json b/crates/screenpipe-semantic/evals/context/cases.json
index cf220f456..a63b0563b 100644
--- a/crates/screenpipe-semantic/evals/context/cases.json
+++ b/crates/screenpipe-semantic/evals/context/cases.json
@@ -27,7 +27,7 @@
{ "role": "AXStaticText", "text": "Alice", "depth": 3, "class_name": "sender" },
{ "role": "AXStaticText", "text": "notarization is still blocking", "depth": 3 },
{ "role": "AXButton", "text": "Add reaction", "depth": 2 },
- { "role": "AXStaticText", "text": "stale hidden Slack instruction", "depth": 1, "on_screen": false }
+ { "role": "AXStaticText", "text": "stale hidden Slack instruction", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }
]
},
{
@@ -60,7 +60,7 @@
{ "role": "AXStaticText", "text": "ChatGPT said:", "depth": 3 },
{ "role": "AXStaticText", "text": "Use a bounded replay eval", "depth": 2 },
{ "role": "AXButton", "text": "Copy response", "depth": 2 },
- { "role": "AXStaticText", "text": "stale hidden ChatGPT answer", "depth": 1, "on_screen": false }
+ { "role": "AXStaticText", "text": "stale hidden ChatGPT answer", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }
]
},
{
@@ -94,7 +94,7 @@
{ "role": "AXGroup", "text": "", "depth": 3, "class_name": "font-claude-response relative" },
{ "role": "AXStaticText", "text": "Use exact DOM message boundaries.", "depth": 4 },
{ "role": "AXButton", "text": "Copy", "depth": 3 },
- { "role": "AXStaticText", "text": "stale hidden Claude answer", "depth": 1, "on_screen": false }
+ { "role": "AXStaticText", "text": "stale hidden Claude answer", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }
]
},
{
@@ -130,7 +130,7 @@
"value": "extract the active note\nreplay archived notes",
"class_name": "cm-content cm-lineWrapping"
},
- { "role": "AXTextArea", "text": "stale hidden Obsidian note", "depth": 1, "value": "stale hidden Obsidian note", "on_screen": false }
+ { "role": "AXTextArea", "text": "stale hidden Obsidian note", "depth": 1, "value": "stale hidden Obsidian note", "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }
]
},
{
@@ -157,7 +157,7 @@
{ "role": "Group", "text": "", "depth": 3, "class_name": "message-body" },
{ "role": "Text", "text": "Please retry signing.", "depth": 4 },
{ "role": "Button", "text": "Archive", "depth": 2 },
- { "role": "Text", "text": "stale hidden Gmail message", "depth": 1, "on_screen": false }
+ { "role": "Text", "text": "stale hidden Gmail message", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }
]
},
{
@@ -185,7 +185,7 @@
"value": "Ship semantic context\nMeasure token reduction",
"help_text": "Launch plan"
},
- { "role": "AXStaticText", "text": "stale hidden Notion paragraph", "depth": 1, "on_screen": false }
+ { "role": "AXStaticText", "text": "stale hidden Notion paragraph", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }
]
},
{
@@ -209,7 +209,7 @@
{ "role": "Group", "text": "", "depth": 1, "class_name": "task-item completed" },
{ "role": "Text", "text": "Define schema", "depth": 2, "class_name": "task-title" },
{ "role": "Button", "text": "Add task", "depth": 1 },
- { "role": "Text", "text": "stale hidden Todoist task", "depth": 1, "on_screen": false }
+ { "role": "Text", "text": "stale hidden Todoist task", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }
]
},
{
@@ -243,7 +243,7 @@
},
{ "role": "AXStaticText", "text": "Parser review", "depth": 2, "class_name": "event-title" },
{ "role": "AXButton", "text": "Today", "depth": 1 },
- { "role": "AXStaticText", "text": "stale hidden Calendar event", "depth": 1, "on_screen": false }
+ { "role": "AXStaticText", "text": "stale hidden Calendar event", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }
]
},
{
@@ -266,7 +266,7 @@
{ "role": "AXStaticText", "text": "$ cargo test", "depth": 1 },
{ "role": "AXStaticText", "text": "24 passed", "depth": 1 },
{ "role": "AXButton", "text": "New Tab", "depth": 1 },
- { "role": "AXStaticText", "text": "stale hidden terminal command", "depth": 1, "on_screen": false }
+ { "role": "AXStaticText", "text": "stale hidden terminal command", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }
]
},
{
@@ -301,7 +301,7 @@
{ "role": "AXStaticText", "text": "17 passed", "depth": 3 },
{ "role": "AXButton", "text": "Explorer", "depth": 2 },
{ "role": "AXButton", "text": "Search", "depth": 2 },
- { "role": "AXStaticText", "text": "stale hidden editor buffer", "depth": 1, "on_screen": false }
+ { "role": "AXStaticText", "text": "stale hidden editor buffer", "depth": 1, "on_screen": false, "bounds": { "left": 0.1, "top": -0.9, "width": 0.6, "height": 0.15 } }
]
}
]
diff --git a/crates/screenpipe-semantic/examples/replay.rs b/crates/screenpipe-semantic/examples/replay.rs
index a688f0284..8ce29b7d7 100644
--- a/crates/screenpipe-semantic/examples/replay.rs
+++ b/crates/screenpipe-semantic/examples/replay.rs
@@ -4,8 +4,8 @@
use screenpipe_semantic::{
adapt_captured_accessibility_tree, parsers::builtin_parser_registry, render_semantic_context,
- AppIdentity, CapturedAccessibilityNode, OutputBudget, ParseContext, ParserRegistry, Platform,
- TreeBudget, ValidatedParseOutcome,
+ AppIdentity, CapturedAccessibilityNode, NodeBounds, OutputBudget, ParseContext, ParserRegistry,
+ Platform, TreeBudget, ValidatedParseOutcome,
};
use serde::{Deserialize, Serialize};
use std::io::{BufRead, Read};
@@ -295,6 +295,14 @@ mod tests {
role: "AXStaticText".into(),
text: "private offscreen sentinel".into(),
on_screen: Some(false),
+ // Off-window with known geometry: suppression requires that
+ // evidence, so keep the sentinel in the suppressed shape.
+ bounds: Some(NodeBounds {
+ left: 0.1,
+ top: -0.9,
+ width: 0.6,
+ height: 0.15,
+ }),
..Default::default()
},
];
diff --git a/crates/screenpipe-semantic/src/capture.rs b/crates/screenpipe-semantic/src/capture.rs
index 71d07dd1f..4fc2683e3 100644
--- a/crates/screenpipe-semantic/src/capture.rs
+++ b/crates/screenpipe-semantic/src/capture.rs
@@ -99,6 +99,16 @@ pub struct AdaptedSemanticTree {
/// off-screen retain structural fields and flags, but human-readable content is
/// withheld so scrollback and hidden overflow cannot become semantic memory.
/// Unknown visibility remains fail-open.
+///
+/// Suppression additionally requires retained geometry, because `on_screen`
+/// alone does not distinguish the two ways a node can miss the window. A node
+/// with bounds that lie off-window is scrollback — exactly what suppression is
+/// for. A node with no bounds at all renders no pixels anywhere and tells us
+/// nothing about where its content sits; screen-reader affordances
+/// (`aria-live` regions, visually-hidden labels) land here and describe what
+/// IS on screen, so treating them as scrollback deleted visible content. On
+/// real Windows captures that cost ~85% of Claude desktop conversation
+/// extraction. No geometry is therefore unknown visibility, and fails open.
pub fn adapt_captured_accessibility_tree(
nodes: &[CapturedAccessibilityNode],
budget: TreeBudget,
@@ -152,7 +162,7 @@ pub fn adapt_captured_accessibility_tree(
class_count += 1;
}
let classes = &class_buffer[..class_count];
- let known_offscreen = node.on_screen == Some(false);
+ let known_offscreen = node.on_screen == Some(false) && node.bounds.is_some();
let source_description = node
.help_text
.as_deref()
diff --git a/crates/screenpipe-semantic/src/parsers/families.rs b/crates/screenpipe-semantic/src/parsers/families.rs
index 8ffa0e2a6..f6de600e5 100644
--- a/crates/screenpipe-semantic/src/parsers/families.rs
+++ b/crates/screenpipe-semantic/src/parsers/families.rs
@@ -650,9 +650,14 @@ fn mail_row_cell_text<'a>(tree: &'a SemanticTree, row: NodeId, class: &str) -> O
struct LabeledTurns {
title: String,
- messages: Vec<(NodeId, String, String)>,
+ /// Same shape the marker path produces: node, actor, actor evidence,
+ /// time label, body.
+ messages: Vec<(NodeId, String, &'static str, Option<String>, String)>,
}
+/// Actor evidence for turns identified by their per-turn action controls
+/// rather than by direction state or an explicit author label.
+const CONTROL_ANCHOR: &str = "control_anchor";
const UIA_CONTENT_ROLES: &[&str] = &["Text", "Heading", "Link", "Paragraph", "ListItem"];
const MAX_UIA_BUFFER_NODES: usize = 1024;
const MAX_UIA_BODY_BYTES: usize = 48 * 1024;
@@ -692,7 +697,7 @@ fn labeled_uia_turns(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Option
return None;
}
- let mut messages: Vec<(NodeId, String, String)> = Vec::new();
+ let mut messages: Vec<(NodeId, String, &'static str, Option<String>, String)> = Vec::new();
let mut buffer: Vec<NodeId> = Vec::new();
let mut title: Option<String> = None;
let mut last_text: Option<NodeId> = None;
@@ -720,7 +725,7 @@ fn labeled_uia_turns(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Option
buffer.clear();
if let Some(body) = collect_text(tree, node) {
if messages.len() < MAX_STRUCTURAL_CANDIDATES {
- messages.push((node, "[user]".to_owned(), body));
+ messages.push((node, "[user]".to_owned(), CONTROL_ANCHOR, None, body));
}
}
skip_under = Some(node);
@@ -738,7 +743,13 @@ fn labeled_uia_turns(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Option
} else if label.eq_ignore_ascii_case("Good response") {
if !buffer.is_empty() && messages.len() < MAX_STRUCTURAL_CANDIDATES {
if let Some(body) = joined_buffer_text(tree, &buffer) {
- messages.push((buffer[0], profile.display_name.to_owned(), body));
+ messages.push((
+ buffer[0],
+ profile.display_name.to_owned(),
+ CONTROL_ANCHOR,
+ None,
+ body,
+ ));
}
}
buffer.clear();
@@ -754,7 +765,18 @@ fn labeled_uia_turns(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Option
{
continue;
}
- if node_content(tree, node).is_none() {
+ let Some(content) = node_content(tree, node) else {
+ continue;
+ };
+ // These surfaces render the turn time as a bare Text sibling right
+ // after the turn's own controls. Attach it to the turn just closed
+ // and keep it out of the body.
+ if content.len() <= 96 && is_message_time_label(content) {
+ if let Some(last) = messages.last_mut() {
+ last.3.get_or_insert_with(|| {
+ content.replace(['\u{00a0}', '\u{202f}'], " ").to_owned()
+ });
+ }
continue;
}
if role.eq_ignore_ascii_case("Text") {
diff --git a/crates/screenpipe-semantic/tests/capture_adapter.rs b/crates/screenpipe-semantic/tests/capture_adapter.rs
index fd6f264b1..2d569f594 100644
--- a/crates/screenpipe-semantic/tests/capture_adapter.rs
+++ b/crates/screenpipe-semantic/tests/capture_adapter.rs
@@ -105,6 +105,14 @@ fn suppresses_known_offscreen_content_without_dropping_structure() {
value: Some("hidden value".into()),
help_text: Some("hidden description".into()),
on_screen: Some(false),
+ // Scrolled above the viewport: known geometry, off-window. This is
+ // the shape suppression exists for.
+ bounds: Some(NodeBounds {
+ left: 0.1,
+ top: -0.8,
+ width: 0.5,
+ height: 0.2,
+ }),
automation_id: Some("message-container".into()),
class_name: Some("message assistant".into()),
..Default::default()
@@ -138,6 +146,32 @@ fn suppresses_known_offscreen_content_without_dropping_structure() {
assert_eq!(adapted.stats.value_nodes, 0);
}
+#[test]
+fn keeps_content_for_offscreen_nodes_that_have_no_geometry() {
+ // Screen-reader affordances (aria-live announcements, visually-hidden
+ // labels) render no pixels anywhere, so the platform reports them as not
+ // on screen with no usable rect. They describe content that IS visible,
+ // so they are unknown visibility rather than scrollback, and fail open.
+ let nodes = vec![CapturedAccessibilityNode {
+ role: "AXStaticText".into(),
+ text: "You said: ship the parser".into(),
+ on_screen: Some(false),
+ bounds: None,
+ class_name: Some("sr-only select-none".into()),
+ ..Default::default()
+ }];
+ let adapted = adapt_captured_accessibility_tree(&nodes, TreeBudget::default()).unwrap();
+ let root = adapted.tree.roots().next().unwrap();
+
+ assert_eq!(adapted.tree.text(root), Some("You said: ship the parser"));
+ assert_eq!(adapted.stats.known_offscreen_nodes, 0);
+ assert_eq!(adapted.stats.suppressed_offscreen_content_nodes, 0);
+ // Structure and flags still record what the platform reported.
+ let flags = adapted.tree.flags(root).unwrap();
+ assert_ne!(flags & CapturedNodeFlags::ON_SCREEN_KNOWN, 0);
+ assert_eq!(flags & CapturedNodeFlags::ON_SCREEN, 0);
+}
+
#[test]
fn keeps_content_when_visibility_is_unknown() {
let nodes = vec![CapturedAccessibilityNode {