YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat: add four app parsers and improve context quality (#7167)
Louis Beaumont073b9d31ad87
diff --git a/crates/screenpipe-a11y/examples/semantic_capture_probe.rs b/crates/screenpipe-a11y/examples/semantic_capture_probe.rsindex 3f4813336..5b3b556c5 100644--- a/crates/screenpipe-a11y/examples/semantic_capture_probe.rs+++ b/crates/screenpipe-a11y/examples/semantic_capture_probe.rs@@ -18,21 +18,19 @@//!//! Usage://! cargo run -p screenpipe-a11y --example semantic_capture_probe -- \-//! [--delay-secs 5] [--samples 1] [--interval-ms 1500]+//! [--delay-secs 5] [--samples 1] [--interval-ms 1500] [--semantic-off]-#[cfg(target_os = "windows")]+#[cfg(any(target_os = "windows", target_os = "macos"))]fn main() {probe::run();}-#[cfg(not(target_os = "windows"))]+#[cfg(not(any(target_os = "windows", target_os = "macos")))]fn main() {- // The walker itself is cross-platform, but the probe's foreground-focus- // driver is Windows-only today.- eprintln!("semantic_capture_probe currently targets Windows");+ eprintln!("semantic_capture_probe currently targets Windows and macOS");}-#[cfg(target_os = "windows")]+#[cfg(any(target_os = "windows", target_os = "macos"))]mod probe {use screenpipe_a11y::tree::{create_tree_walker, AccessibilityTreeNode, TreeSnapshot, TreeWalkResult, TreeWalkerConfig,@@ -51,12 +49,13 @@ mod probe {let delay_secs = opt("--delay-secs").unwrap_or(5);let samples = opt("--samples").unwrap_or(1).max(1);let interval_ms = opt("--interval-ms").unwrap_or(1500);+ let capture_semantic_structure = !args.iter().any(|arg| arg == "--semantic-off");std::thread::sleep(Duration::from_secs(delay_secs));let walker = create_tree_walker(TreeWalkerConfig {capture_app_identity: true,- capture_semantic_structure: true,+ capture_semantic_structure,..Default::default()});@@ -80,8 +79,14 @@ mod probe {.map(|since| since.as_millis() as i64).unwrap_or_default(),"content_hash": snapshot.content_hash,+ "capture": {+ "truncated": snapshot.truncated,+ "truncation_reason": format!("{:?}", snapshot.truncation_reason),+ "node_count": snapshot.node_count,+ "walk_duration_ms": snapshot.walk_duration.as_millis(),+ },"app": {- "platform": "windows",+ "platform": std::env::consts::OS,"app_id": snapshot.app_id,"executable": snapshot.executable,"display_name": snapshot.app_name,@@ -116,6 +121,9 @@ mod probe {node.bounds.as_ref().map(|b| json!(b)),);insert_some(&mut object, "on_screen", node.on_screen.map(Value::from));+ if node.semantic_offscreen {+ object.insert("offscreen_geometry".into(), Value::Bool(true));+ }insert_some(&mut object,"automation_id",diff --git a/crates/screenpipe-a11y/src/platform/windows_uia.rs b/crates/screenpipe-a11y/src/platform/windows_uia.rsindex f22de45f9..8c1dc2724 100644--- a/crates/screenpipe-a11y/src/platform/windows_uia.rs+++ b/crates/screenpipe-a11y/src/platform/windows_uia.rs@@ -213,6 +213,10 @@ pub(crate) struct UiaContext {/// Uses Full mode so walked elements retain live COM references for navigation.walker_cache_request: IUIAutomationCacheRequest,tree_walker: IUIAutomationTreeWalker,+ /// RawView is reserved for opt-in semantic capture. Chromium can omit+ /// inert HTML containers (and therefore their classes/ids) from ControlView.+ semantic_cache_request: IUIAutomationCacheRequest,+ semantic_tree_walker: IUIAutomationTreeWalker,}impl UiaContext {@@ -277,6 +281,9 @@ impl UiaContext {walker_cache_request.SetTreeScope(TreeScope_Element)?;let tree_walker = automation.ControlViewWalker()?;+ let semantic_cache_request = walker_cache_request.Clone()?;+ semantic_cache_request.SetTreeFilter(&automation.RawViewCondition()?)?;+ let semantic_tree_walker = automation.RawViewWalker()?;Ok(Self {automation,@@ -284,6 +291,8 @@ impl UiaContext {enrichment_cache_request,walker_cache_request,tree_walker,+ semantic_cache_request,+ semantic_tree_walker,})}}@@ -398,16 +407,52 @@ impl UiaContext {&self,hwnd: HWND,budget: &mut TreeBudget,+ ) -> Option<AccessibilityNode> {+ self.capture_window_tree_walker_with(+ hwnd,+ budget,+ &self.tree_walker,+ &self.walker_cache_request,+ )+ }++ /// Semantic browser capture needs inert HTML containers that Chromium can+ /// omit from ControlView. RawView is still bounded by the production node+ /// and wall-clock budgets and is never used by the historical capture path.+ pub(crate) fn capture_window_tree_bounded_semantic(+ &self,+ hwnd: HWND,+ max_elements: usize,+ timeout: Duration,+ ) -> Option<CapturedTree> {+ let deadline = Instant::now() + timeout;+ let mut budget = TreeBudget::new(max_elements, deadline);+ self.capture_window_tree_walker_with(+ hwnd,+ &mut budget,+ &self.semantic_tree_walker,+ &self.semantic_cache_request,+ )+ .map(|root| CapturedTree {+ root,+ truncation: budget.truncation,+ })+ }++ fn capture_window_tree_walker_with(+ &self,+ hwnd: HWND,+ budget: &mut TreeBudget,+ tree_walker: &IUIAutomationTreeWalker,+ cache_request: &IUIAutomationCacheRequest,) -> Option<AccessibilityNode> {unsafe {// Get a live element (required for TreeWalker navigation)let live_element = self.automation.ElementFromHandle(hwnd).ok()?;// Cache properties on the root element- let cached_root = live_element- .BuildUpdatedCache(&self.walker_cache_request)- .ok()?;+ let cached_root = live_element.BuildUpdatedCache(cache_request).ok()?;- Some(self.build_node_walker(&cached_root, budget))+ Some(self.build_node_walker(&cached_root, budget, tree_walker, cache_request))}}@@ -419,6 +464,8 @@ impl UiaContext {&self,element: &IUIAutomationElement,budget: &mut TreeBudget,+ tree_walker: &IUIAutomationTreeWalker,+ cache_request: &IUIAutomationCacheRequest,) -> AccessibilityNode {budget.count += 1;@@ -443,20 +490,26 @@ impl UiaContext {if !budget.exhausted() {unsafe {// Navigate to first child via TreeWalker- if let Ok(child) = self- .tree_walker- .GetFirstChildElementBuildCache(element, &self.walker_cache_request)+ if let Ok(child) =+ tree_walker.GetFirstChildElementBuildCache(element, cache_request){- children.push(self.build_node_walker(&child, budget));+ children.push(self.build_node_walker(+ &child,+ budget,+ tree_walker,+ cache_request,+ ));// Iterate siblingslet mut current = child;while !budget.exhausted() {- match self- .tree_walker- .GetNextSiblingElementBuildCache(¤t, &self.walker_cache_request)- {+ match tree_walker.GetNextSiblingElementBuildCache(¤t, cache_request) {Ok(next) => {- children.push(self.build_node_walker(&next, budget));+ children.push(self.build_node_walker(+ &next,+ budget,+ tree_walker,+ cache_request,+ ));current = next;}Err(_) => break,diff --git a/crates/screenpipe-a11y/src/tree/macos.rs b/crates/screenpipe-a11y/src/tree/macos.rsindex 0e600eac7..15f2aaec1 100644--- a/crates/screenpipe-a11y/src/tree/macos.rs+++ b/crates/screenpipe-a11y/src/tree/macos.rs@@ -1895,6 +1895,7 @@ fn capture_structural_node(| "AXScrollArea"| "AXList"| "AXRow"+ | "AXColumn"| "AXTable"| "AXWebArea"| "AXDocumentArticle"@@ -1909,6 +1910,7 @@ fn capture_structural_node(role_str,"AXWindow"| "AXRow"+ | "AXColumn"| "AXCell"| "AXWebArea"| "AXDocumentArticle"@@ -1943,6 +1945,8 @@ fn capture_structural_node(let mut node =AccessibilityTreeNode::new(role_str.to_owned(), text, depth.min(255) as u8, bounds);node.on_screen = on_screen;+ node.semantic_offscreen =+ state.capture_parser_structure && semantic_offscreen(frame, on_screen);node.walk_index = state.node_count.min(u32::MAX as usize) as u32;node.semantic_only = true;node.automation_relevant = automation_relevant;@@ -2218,6 +2222,8 @@ fn extract_text(bounds.clone(),);node.on_screen = on_screen;+ node.semantic_offscreen =+ state.capture_parser_structure && semantic_offscreen(frame, on_screen);node.walk_index = state.node_count.min(u32::MAX as usize) as u32;node.value = Some(trimmed.clone());fill_ax_props(&mut node, elem, role_str, state.capture_semantic_structure);@@ -2246,6 +2252,8 @@ fn extract_text(bounds.clone(),);node.on_screen = on_screen;+ node.semantic_offscreen =+ state.capture_parser_structure && semantic_offscreen(frame, on_screen);node.walk_index = state.node_count.min(u32::MAX as usize) as u32;fill_ax_props(&mut node, elem, role_str, state.capture_semantic_structure);apply_primary_semantic_attrs(&mut node, attrs);@@ -2267,6 +2275,8 @@ fn extract_text(bounds,);node.on_screen = on_screen;+ node.semantic_offscreen =+ state.capture_parser_structure && semantic_offscreen(frame, on_screen);node.walk_index = state.node_count.min(u32::MAX as usize) as u32;fill_ax_props(&mut node, elem, role_str, state.capture_semantic_structure);apply_primary_semantic_attrs(&mut node, attrs);@@ -2286,6 +2296,8 @@ fn extract_text(bounds,);node.on_screen = on_screen;+ node.semantic_offscreen =+ state.capture_parser_structure && semantic_offscreen(frame, on_screen);node.walk_index = state.node_count.min(u32::MAX as usize) as u32;fill_ax_props(&mut node, elem, role_str, state.capture_semantic_structure);apply_primary_semantic_attrs(&mut node, attrs);@@ -2368,6 +2380,15 @@ fn is_on_screen())}+/// Keep proof of off-window geometry before screenshot normalization drops it.+/// Zero-sized accessibility labels remain unknown, preserving screen-reader text.+fn semantic_offscreen(frame: Option<(f64, f64, f64, f64)>, on_screen: Option<bool>) -> bool {+ on_screen == Some(false)+ && frame.is_some_and(|(x, y, w, h)| {+ x.is_finite() && y.is_finite() && w.is_finite() && h.is_finite() && w > 0.0 && h > 0.0+ })+}+/// Normalize an element's screen-absolute frame to 0-1 coordinates.////// Prefers **monitor-relative** normalization (matching the full-screen capture image).@@ -2796,6 +2817,24 @@ fn fill_ax_props(mod tests {use super::*;+ #[test]+ fn retains_offwindow_geometry_without_suppressing_zero_sized_labels() {+ assert!(semantic_offscreen(+ Some((10.0, 2000.0, 80.0, 20.0)),+ Some(false)+ ));+ assert!(!semantic_offscreen(+ Some((10.0, 20.0, 80.0, 20.0)),+ Some(true)+ ));+ assert!(!semantic_offscreen(Some((0.0, 0.0, 0.0, 0.0)), Some(false)));+ assert!(!semantic_offscreen(+ Some((f64::NAN, 0.0, 80.0, 20.0)),+ Some(false)+ ));+ assert!(!semantic_offscreen(None, Some(false)));+ }+#[test]fn apple_screenshot_ui_is_excluded_from_walks() {// Both spellings reach `is_excluded_app` depending on whether the piddiff --git a/crates/screenpipe-a11y/src/tree/mod.rs b/crates/screenpipe-a11y/src/tree/mod.rsindex 970010393..2a04e1d19 100644--- a/crates/screenpipe-a11y/src/tree/mod.rs+++ b/crates/screenpipe-a11y/src/tree/mod.rs@@ -194,6 +194,10 @@ pub struct AccessibilityTreeNode {/// Space-delimited parser-only DOM classes.#[serde(skip)]pub semantic_dom_classes: Option<String>,+ /// Positive-sized native geometry lies outside the window, even when+ /// screenshot-normalized bounds were discarded. Parser-only evidence.+ #[serde(skip)]+ pub semantic_offscreen: bool,}impl AccessibilityTreeNode {@@ -228,6 +232,7 @@ impl AccessibilityTreeNode {semantic_description: None,semantic_dom_identifier: None,semantic_dom_classes: None,+ semantic_offscreen: false,}}}@@ -1062,6 +1067,7 @@ mod tests {node.semantic_description = Some("parser description".into());node.semantic_dom_identifier = Some("message-list".into());node.semantic_dom_classes = Some("message selected".into());+ node.semantic_offscreen = true;let json = serde_json::to_string(&node).expect("serialize persisted node");assert!(!json.contains("walk_index"));@@ -1069,6 +1075,7 @@ mod tests {assert!(!json.contains("parser description"));assert!(!json.contains("message-list"));assert!(!json.contains("message selected"));+ assert!(!json.contains("semantic_offscreen"));}#[test]diff --git a/crates/screenpipe-a11y/src/tree/windows.rs b/crates/screenpipe-a11y/src/tree/windows.rsindex 87f5bd98b..1c92cb932 100644--- a/crates/screenpipe-a11y/src/tree/windows.rs+++ b/crates/screenpipe-a11y/src/tree/windows.rs@@ -112,6 +112,25 @@ fn is_automation_structure_type(control_type: &str) -> bool {)}+/// UIA containers that preserve the browser document hierarchy needed by the+/// semantic parsers. Text-bearing controls are emitted by the normal path;+/// this list only covers otherwise-discarded structural nodes.+fn is_semantic_structure_type(control_type: &str) -> bool {+ matches!(+ control_type,+ "Document" | "Group" | "Pane" | "Custom" | "List" | "Table" | "DataGrid"+ )+}++fn is_semantic_raw_view_app(app_lower: &str) -> bool {+ [+ "edge", "chrome", "chromium", "brave", "firefox", "slack", "discord", "claude", "obsidian",+ "code", "cursor", "windsurf", "notion",+ ]+ .iter()+ .any(|name| app_lower.contains(name))+}+/// Lazily-initialized COM + UIA state. Wrapped in `UnsafeCell` because/// `TreeWalkerPlatform::walk_focused_window` takes `&self` but we need/// to mutate on first call (lazy init). The walker is single-threaded.@@ -349,11 +368,16 @@ impl TreeWalkerPlatform for WindowsTreeWalker {// fallback (Chromium/Electron), which is serviced synchronously on the// target app's UI thread and freezes it for the duration of the walk.let remaining_budget = effective_timeout.saturating_sub(start.elapsed());- let captured =- match uia.capture_window_tree_bounded(hwnd, effective_max_nodes, remaining_budget) {- Some(captured) => captured,- None => return Ok(TreeWalkResult::NotFound),- };+ let captured = match if self.config.capture_semantic_structure+ && is_semantic_raw_view_app(&app_lower)+ {+ uia.capture_window_tree_bounded_semantic(hwnd, effective_max_nodes, remaining_budget)+ } else {+ uia.capture_window_tree_bounded(hwnd, effective_max_nodes, remaining_budget)+ } {+ Some(captured) => captured,+ None => return Ok(TreeWalkResult::NotFound),+ };let root = captured.root;let truncation_reason = captured.truncation;@@ -367,6 +391,8 @@ impl TreeWalkerPlatform for WindowsTreeWalker {// Extract text from the tree (matching macOS text extraction behavior)let mut text_buffer = String::with_capacity(4096);let mut nodes = Vec::with_capacity(256);+ let mut semantic_nodes = Vec::with_capacity(128);+ let mut walk_index = 0u32;let mut browser_url: Option<String> = None;let mut hit_ignored_extension = false;extract_text_from_tree(@@ -381,7 +407,11 @@ impl TreeWalkerPlatform for WindowsTreeWalker {ignored_patterns.as_ref(),&app_lower,&mut hit_ignored_extension,+ self.config.capture_semantic_structure,self.config.capture_automation_structure,+ false,+ &mut semantic_nodes,+ &mut walk_index,);if hit_ignored_extension {@@ -465,7 +495,7 @@ impl TreeWalkerPlatform for WindowsTreeWalker {window_name,text_content: text_buffer,nodes,- semantic_nodes: Vec::new(),+ semantic_nodes,browser_url,document_path,timestamp: Utc::now(),@@ -665,6 +695,23 @@ fn normalize_bounds(})}+/// Preserve trustworthy native visibility evidence even when normalized bounds+/// are omitted because the element is wholly outside the monitor.+fn semantic_offscreen(+ bounds: Option<&crate::events::ElementBounds>,+ on_screen: Option<bool>,+) -> bool {+ on_screen == Some(false)+ && bounds.is_some_and(|bounds| {+ bounds.x.is_finite()+ && bounds.y.is_finite()+ && bounds.width.is_finite()+ && bounds.height.is_finite()+ && bounds.width > 0.0+ && bounds.height > 0.0+ })+}+/// Build an AccessibilityTreeNode from a UIA AccessibilityNode, propagating automation properties.fn make_tree_node(uia_node: &AccessibilityNode,@@ -673,6 +720,8 @@ fn make_tree_node(depth: usize,bounds: Option<NodeBounds>,on_screen: Option<bool>,+ capture_semantic_dom: bool,+ walk_index: u32,) -> AccessibilityTreeNode {let mut n = AccessibilityTreeNode::new(role.to_string(),@@ -681,6 +730,7 @@ fn make_tree_node(bounds,);n.on_screen = on_screen;+ n.walk_index = walk_index;n.automation_id = uia_node.automation_id.clone();n.class_name = uia_node.class_name.clone();n.value = uia_node.value.clone();@@ -694,6 +744,11 @@ fn make_tree_node(n.accelerator_key = uia_node.accelerator_key.clone();n.access_key = uia_node.access_key.clone();n.role_description = uia_node.localized_control_type.clone();+ if capture_semantic_dom {+ n.semantic_dom_identifier = uia_node.automation_id.clone();+ n.semantic_dom_classes = uia_node.class_name.clone();+ n.semantic_offscreen = semantic_offscreen(uia_node.bounds.as_ref(), on_screen);+ }n}@@ -712,13 +767,21 @@ fn extract_text_from_tree(ignored_patterns: &[WindowPattern],focused_app_lower: &str,hit_ignored_extension: &mut bool,+ capture_semantic_structure: bool,capture_automation_structure: bool,+ inside_document: bool,+ semantic_nodes: &mut Vec<AccessibilityTreeNode>,+ walk_index: &mut u32,) {if depth > max_depth {return;}let ct = node.control_type.as_str();+ let current_walk_index = *walk_index;+ *walk_index = walk_index.saturating_add(1);+ let in_document = inside_document || ct.eq_ignore_ascii_case("Document");+ let capture_semantic_dom = capture_semantic_structure && in_document;// Skip decorative elementsif SKIP_TYPES.iter().any(|&s| ct.eq_ignore_ascii_case(s))@@ -750,8 +813,16 @@ fn extract_text_from_tree(// Never extract the value of password fieldsif node.is_password == Some(true) {if capture_automation_structure {- let mut tree_node =- make_tree_node(node, ct, "", depth, norm_bounds.clone(), on_screen);+ let mut tree_node = make_tree_node(+ node,+ ct,+ "",+ depth,+ norm_bounds.clone(),+ on_screen,+ false,+ current_walk_index,+ );// UIA Value can contain the secret even when IsPassword is// set. Preserve targeting metadata, never the field value.tree_node.value = None;@@ -770,6 +841,8 @@ fn extract_text_from_tree(depth,norm_bounds.clone(),on_screen,+ capture_semantic_dom,+ current_walk_index,));// Don't recurse into text controls — their children are sub-elements of the same textreturn;@@ -832,6 +905,8 @@ fn extract_text_from_tree(depth,norm_bounds.clone(),on_screen,+ capture_semantic_dom,+ current_walk_index,));}}@@ -849,6 +924,8 @@ fn extract_text_from_tree(depth,norm_bounds.clone(),on_screen,+ capture_semantic_dom,+ current_walk_index,));}}@@ -869,6 +946,8 @@ fn extract_text_from_tree(depth,norm_bounds.clone(),on_screen,+ capture_semantic_dom,+ current_walk_index,));}} else if ct.eq_ignore_ascii_case("Custom") {@@ -883,6 +962,8 @@ fn extract_text_from_tree(depth,norm_bounds.clone(),on_screen,+ capture_semantic_dom,+ current_walk_index,));}}@@ -901,11 +982,47 @@ fn extract_text_from_tree(.map(str::trim).filter(|name| !name.is_empty()).unwrap_or("");- let mut tree_node = make_tree_node(node, ct, text, depth, norm_bounds.clone(), on_screen);+ let mut tree_node = make_tree_node(+ node,+ ct,+ text,+ depth,+ norm_bounds.clone(),+ on_screen,+ capture_semantic_dom,+ current_walk_index,+ );tree_node.automation_relevant = true;nodes.push(tree_node);}+ if capture_semantic_dom+ && nodes.len() == nodes_before+ && (is_semantic_structure_type(ct)+ || node+ .automation_id+ .as_deref()+ .is_some_and(|id| !id.trim().is_empty())+ || node+ .class_name+ .as_deref()+ .is_some_and(|class| !class.trim().is_empty()))+ {+ let text = node.name.as_deref().map(str::trim).unwrap_or("");+ let mut tree_node = make_tree_node(+ node,+ ct,+ text,+ depth,+ norm_bounds.clone(),+ on_screen,+ true,+ current_walk_index,+ );+ tree_node.semantic_only = true;+ semantic_nodes.push(tree_node);+ }+// Recurse into childrenfor child in &node.children {extract_text_from_tree(@@ -920,7 +1037,11 @@ fn extract_text_from_tree(ignored_patterns,focused_app_lower,hit_ignored_extension,+ capture_semantic_structure,capture_automation_structure,+ in_document,+ semantic_nodes,+ walk_index,);}}@@ -977,6 +1098,45 @@ fn append_text(buffer: &mut String, text: &str) {mod tests {use super::*;+ #[allow(clippy::too_many_arguments)]+ fn extract_for_test(+ node: &AccessibilityNode,+ depth: usize,+ max_depth: usize,+ buffer: &mut String,+ nodes: &mut Vec<AccessibilityTreeNode>,+ browser_url: &mut Option<String>,+ monitor_rect: &Option<MonitorRect>,+ window_rect: &Option<WindowRect>,+ ignored_patterns: &[WindowPattern],+ focused_app_lower: &str,+ hit_ignored_extension: &mut bool,+ capture_semantic_structure: bool,+ capture_automation_structure: bool,+ ) -> Vec<AccessibilityTreeNode> {+ let mut semantic_nodes = Vec::new();+ let mut walk_index = 0;+ extract_text_from_tree(+ node,+ depth,+ max_depth,+ buffer,+ nodes,+ browser_url,+ monitor_rect,+ window_rect,+ ignored_patterns,+ focused_app_lower,+ hit_ignored_extension,+ capture_semantic_structure,+ capture_automation_structure,+ false,+ &mut semantic_nodes,+ &mut walk_index,+ );+ semantic_nodes+ }+#[test]fn test_skip_types() {assert!(SKIP_TYPES@@ -1038,7 +1198,7 @@ mod tests {let mut buf = String::new();let mut nodes = Vec::new();let mut url = None;- extract_text_from_tree(+ extract_for_test(&tree,0,10,@@ -1051,6 +1211,7 @@ mod tests {"",&mut false,false,+ false,);// Text node's name should be captured@@ -1098,7 +1259,7 @@ mod tests {let mut buf = String::new();let mut nodes = Vec::new();let mut url = None;- extract_text_from_tree(+ extract_for_test(&tree,0,10,@@ -1110,6 +1271,7 @@ mod tests {&[],"",&mut false,+ false,true,);@@ -1122,6 +1284,83 @@ mod tests {assert!(!buf.contains("must-never-escape"));}+ #[test]+ fn semantic_offscreen_requires_positive_finite_native_geometry() {+ use crate::events::ElementBounds;++ let positive = ElementBounds {+ x: -500.0,+ y: 20.0,+ width: 120.0,+ height: 30.0,+ };+ assert!(semantic_offscreen(Some(&positive), Some(false)));+ assert!(!semantic_offscreen(Some(&positive), Some(true)));++ for invalid in [+ ElementBounds {+ width: 0.0,+ ..positive.clone()+ },+ ElementBounds {+ height: 0.0,+ ..positive.clone()+ },+ ElementBounds {+ x: f64::NAN,+ ..positive.clone()+ },+ ElementBounds {+ width: f64::INFINITY,+ ..positive.clone()+ },+ ] {+ assert!(!semantic_offscreen(Some(&invalid), Some(false)));+ }+ assert!(!semantic_offscreen(None, Some(false)));+ }++ #[test]+ fn combined_semantic_and_automation_capture_keeps_dom_evidence() {+ let tree = AccessibilityNode {+ control_type: "Document".into(),+ children: vec![AccessibilityNode {+ control_type: "Button".into(),+ automation_id: Some("submit".into()),+ class_name: Some("primary action".into()),+ ..Default::default()+ }],+ ..Default::default()+ };+ let mut nodes = Vec::new();+ let semantic = extract_for_test(+ &tree,+ 0,+ 10,+ &mut String::new(),+ &mut nodes,+ &mut None,+ &None,+ &None,+ &[],+ "",+ &mut false,+ true,+ true,+ );++ assert!(+ semantic.is_empty(),+ "automation copy suppresses a duplicate"+ );+ let button = nodes.iter().find(|node| node.role == "Button").unwrap();+ assert_eq!(button.semantic_dom_identifier.as_deref(), Some("submit"));+ assert_eq!(+ button.semantic_dom_classes.as_deref(),+ Some("primary action")+ );+ }+#[test]fn test_document_node_recurses_into_children() {use crate::events::AccessibilityNode;@@ -1159,7 +1398,7 @@ mod tests {let mut buf = String::new();let mut nodes = Vec::new();let mut url = None;- extract_text_from_tree(+ extract_for_test(&tree,0,30,@@ -1172,6 +1411,7 @@ mod tests {"",&mut false,false,+ false,);// URL should be captured as browser_url, NOT as text@@ -1211,6 +1451,86 @@ mod tests {);}+ #[test]+ fn test_semantic_capture_retains_native_browser_dom_evidence_only_when_enabled() {+ let tree = AccessibilityNode {+ control_type: "Window".into(),+ children: vec![AccessibilityNode {+ control_type: "Document".into(),+ value: Some("https://example.com/item".into()),+ children: vec![AccessibilityNode {+ control_type: "Group".into(),+ automation_id: Some("issue-42-permalink".into()),+ class_name: Some("js-comment timeline-comment".into()),+ children: vec![AccessibilityNode {+ control_type: "Text".into(),+ name: Some("Verified body".into()),+ ..Default::default()+ }],+ ..Default::default()+ }],+ ..Default::default()+ }],+ ..Default::default()+ };++ let mut text = String::new();+ let mut nodes = Vec::new();+ let mut url = None;+ let semantic_nodes = extract_for_test(+ &tree,+ 0,+ 10,+ &mut text,+ &mut nodes,+ &mut url,+ &None,+ &None,+ &[],+ "msedge",+ &mut false,+ true,+ false,+ );++ let container = semantic_nodes+ .iter()+ .find(|node| node.semantic_dom_identifier.as_deref() == Some("issue-42-permalink"))+ .expect("native UIA identifier should be retained for semantic parsing");+ assert_eq!(+ container.semantic_dom_classes.as_deref(),+ Some("js-comment timeline-comment")+ );+ assert!(container.semantic_only);+ assert_eq!(container.walk_index, 2);+ assert!(nodes.iter().any(|node| node.text == "Verified body"));++ let mut off_text = String::new();+ let mut off_nodes = Vec::new();+ let mut off_url = None;+ let semantic_off = extract_for_test(+ &tree,+ 0,+ 10,+ &mut off_text,+ &mut off_nodes,+ &mut off_url,+ &None,+ &None,+ &[],+ "msedge",+ &mut false,+ false,+ false,+ );+ assert!(semantic_off.is_empty());+ assert!(off_nodes+ .iter()+ .all(|node| node.semantic_dom_identifier.is_none()+ && node.semantic_dom_classes.is_none()));+ assert_eq!(off_text, text);+ }+#[test]fn test_extension_popup_ignored_via_child_text() {use crate::events::AccessibilityNode;@@ -1253,7 +1573,7 @@ mod tests {let mut url = None;let mut hit = false;- extract_text_from_tree(+ extract_for_test(&bitwarden_popup,0,10,@@ -1266,6 +1586,7 @@ mod tests {"",&mut hit,false,+ false,);assert!(diff --git a/crates/screenpipe-engine/src/semantic_worker.rs b/crates/screenpipe-engine/src/semantic_worker.rsindex 00ef95841..64bb1059d 100644--- a/crates/screenpipe-engine/src/semantic_worker.rs+++ b/crates/screenpipe-engine/src/semantic_worker.rs@@ -660,6 +660,7 @@ fn captured_node(node: &AccessibilityTreeNode, redact_pii: bool) -> CapturedAcceheight: bounds.height,}),on_screen: node.on_screen,+ offscreen_geometry: node.semantic_offscreen,automation_id: node.automation_id.clone(),dom_identifier: node.semantic_dom_identifier.clone(),class_name: nodediff --git a/crates/screenpipe-semantic/evals/quality/README.md b/crates/screenpipe-semantic/evals/quality/README.mdnew file mode 100644index 000000000..d8ddc7fe4--- /dev/null+++ b/crates/screenpipe-semantic/evals/quality/README.md@@ -0,0 +1,37 @@+# Parser context quality++These four synthetic regression cases measure what an agent receives, not+parser execution speed. Their expected values are hand-authored independently+of parser output. They check exact bodies, task state, owners, deadlines,+priority and parent relationships, plus UI-noise exclusion and rendered fields.+Unknown task state stays unknown; due labels are retained verbatim without+inventing a date or timezone. Authored action words and link text stay in bodies.++```sh+cargo test -p screenpipe-semantic --test context_quality --locked+cargo run --release -p screenpipe-semantic --example context_quality --locked+```++The example emits per-case check failures and exact `o200k_base` token counts+for serialized captured nodes and semantic context. To compare revisions, copy+this exact `evals/quality` directory and example into the base checkout before+running it. Do not compare different fixtures or tokenizer versions.++Against base `acdcd8b21d7674e9493c19ad130f54eae9e8283a`, the same fixtures improve+from 16/33 to 33/33 checks and from 207 to 188 context tokens. The 33 checks are+25 exact field/relationship checks, six exclusion checks and two rendered-field+checks. They are regression assertions, not independent facts or a live-app+accuracy score.++| Case | Checks before → after | Context tokens before → after |+|---|---:|---:|+| Slack message actions | 4/8 → 8/8 | 69 → 56 |+| Task decision fields | 2/8 → 8/8 | 19 → 37 |+| Task state evidence | 8/12 → 12/12 | 78 → 62 |+| Mail body actions | 2/5 → 5/5 | 41 → 33 |++The task case needs more tokens because it restores missing owner, due and+priority fields and corrects status. Shorter context alone is not success.+This suite covers known structural contracts, not all app versions or real+Windows/macOS UI behavior. Run the existing context and pipe evals as regression+gates as well; no model calls, cloud credentials or private captures are needed.diff --git a/crates/screenpipe-semantic/evals/quality/cases.json b/crates/screenpipe-semantic/evals/quality/cases.jsonnew file mode 100644index 000000000..c2281783f--- /dev/null+++ b/crates/screenpipe-semantic/evals/quality/cases.json@@ -0,0 +1,457 @@+[+ {+ "id": "slack_message_actions",+ "app": {+ "platform": "windows",+ "display_name": "Slack",+ "executable": "slack.exe",+ "browser_url": null+ },+ "nodes": [+ {+ "role": "Window",+ "depth": 0,+ "text": "#release"+ },+ {+ "role": "Group",+ "depth": 1,+ "text": "",+ "class_name": "message-item"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Alice",+ "class_name": "sender"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Copy link\nShip the fix Friday."+ },+ {+ "role": "Link",+ "depth": 2,+ "text": ""+ },+ {+ "role": "Text",+ "depth": 3,+ "text": "Release checklist"+ },+ {+ "role": "Button",+ "depth": 2,+ "text": ""+ },+ {+ "role": "Text",+ "depth": 3,+ "text": "deployment-notes.pdf"+ },+ {+ "role": "Button",+ "depth": 2,+ "text": ""+ },+ {+ "role": "Text",+ "depth": 3,+ "text": "Reply to thread"+ },+ {+ "role": "Button",+ "depth": 2,+ "text": ""+ },+ {+ "role": "Text",+ "depth": 3,+ "text": "Add reaction"+ },+ {+ "role": "MenuItem",+ "depth": 2,+ "text": ""+ },+ {+ "role": "Text",+ "depth": 3,+ "text": "More actions"+ },+ {+ "role": "Group",+ "depth": 1,+ "text": "",+ "class_name": "message-item outgoing"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Agreed. Keep the label."+ }+ ],+ "checks": [+ {+ "path": "/1/actor",+ "expected": "Alice"+ },+ {+ "path": "/1/body",+ "expected": "Copy link\nShip the fix Friday.\nRelease checklist\ndeployment-notes.pdf"+ },+ {+ "path": "/1/parent_local_id",+ "expected": "conversation"+ },+ {+ "path": "/2/actor",+ "expected": "[user]"+ },+ {+ "path": "/2/body",+ "expected": "Agreed. Keep the label."+ }+ ],+ "forbidden_context": [+ "Reply to thread",+ "Add reaction",+ "More actions"+ ],+ "required_context": []+ },+ {+ "id": "task_decision_fields",+ "app": {+ "platform": "windows",+ "display_name": "Todoist",+ "executable": "Todoist.exe",+ "browser_url": null+ },+ "nodes": [+ {+ "role": "Window",+ "depth": 0,+ "text": "Release tasks"+ },+ {+ "role": "Group",+ "depth": 1,+ "text": "",+ "class_name": "task-item unchecked"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Ship parser context",+ "class_name": "task-title"+ },+ {+ "role": "Group",+ "depth": 2,+ "text": "",+ "class_name": "task-status",+ "role_description": "group"+ },+ {+ "role": "Text",+ "depth": 3,+ "text": "In progress"+ },+ {+ "role": "Group",+ "depth": 2,+ "text": "",+ "class_name": "task-assignee",+ "role_description": "group"+ },+ {+ "role": "Text",+ "depth": 3,+ "text": "Alice"+ },+ {+ "role": "Group",+ "depth": 2,+ "text": "",+ "class_name": "task-due-date",+ "role_description": "group"+ },+ {+ "role": "Text",+ "depth": 3,+ "text": "Friday, September 25"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "High",+ "class_name": "task-priority"+ },+ {+ "role": "Button",+ "depth": 2,+ "text": "Edit due date"+ }+ ],+ "checks": [+ {+ "path": "/0/title",+ "expected": "Ship parser context"+ },+ {+ "path": "/0/status",+ "expected": "In progress"+ },+ {+ "path": "/0/metadata/assignee",+ "expected": "Alice"+ },+ {+ "path": "/0/metadata/due",+ "expected": "Friday, September 25"+ },+ {+ "path": "/0/metadata/priority",+ "expected": "High"+ }+ ],+ "forbidden_context": [+ "Edit due date"+ ],+ "required_context": [+ "assignee=Alice",+ "due=Friday, September 25"+ ]+ },+ {+ "id": "task_state_evidence",+ "app": {+ "platform": "windows",+ "display_name": "Todoist",+ "executable": "Todoist.exe",+ "browser_url": null+ },+ "nodes": [+ {+ "role": "Window",+ "depth": 0,+ "text": "Task states"+ },+ {+ "role": "Group",+ "depth": 1,+ "text": "",+ "class_name": "task-row unchecked",+ "help_text": "Review unchecked migration"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Review unchecked migration",+ "class_name": "task-title"+ },+ {+ "role": "Group",+ "depth": 1,+ "text": "",+ "class_name": "task-row incomplete",+ "help_text": "Complete the rollout"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Complete the rollout",+ "class_name": "task-title"+ },+ {+ "role": "Group",+ "depth": 1,+ "text": "",+ "class_name": "task-row not-completed",+ "help_text": "Review open issues"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Review open issues",+ "class_name": "task-title"+ },+ {+ "role": "Group",+ "depth": 1,+ "text": "",+ "class_name": "task-row completed",+ "help_text": "Write release notes"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Write release notes",+ "class_name": "task-title"+ },+ {+ "role": "Group",+ "depth": 1,+ "text": "",+ "class_name": "task-row is-completed",+ "help_text": "Update handbook"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Update handbook",+ "class_name": "task-title"+ },+ {+ "role": "Group",+ "depth": 1,+ "class_name": "task-row",+ "help_text": "Completed planning last week"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Implement the plan",+ "class_name": "task-title"+ }+ ],+ "checks": [+ {+ "path": "/0/title",+ "expected": "Review unchecked migration"+ },+ {+ "path": "/0/status",+ "expected": null+ },+ {+ "path": "/1/title",+ "expected": "Complete the rollout"+ },+ {+ "path": "/1/status",+ "expected": null+ },+ {+ "path": "/2/title",+ "expected": "Review open issues"+ },+ {+ "path": "/2/status",+ "expected": null+ },+ {+ "path": "/3/title",+ "expected": "Write release notes"+ },+ {+ "path": "/3/status",+ "expected": "completed"+ },+ {+ "path": "/4/title",+ "expected": "Update handbook"+ },+ {+ "path": "/4/status",+ "expected": "completed"+ },+ {+ "path": "/5/title",+ "expected": "Implement the plan"+ },+ {+ "path": "/5/status",+ "expected": null+ }+ ],+ "forbidden_context": [],+ "required_context": []+ },+ {+ "id": "mail_body_actions",+ "app": {+ "platform": "windows",+ "display_name": "Chrome",+ "executable": "chrome.exe",+ "browser_url": "https://mail.google.com/mail/u/0/#inbox/example"+ },+ "nodes": [+ {+ "role": "Window",+ "depth": 0,+ "text": "Inbox"+ },+ {+ "role": "Text",+ "depth": 1,+ "text": "Release approval",+ "class_name": "subject"+ },+ {+ "role": "Group",+ "depth": 1,+ "text": "",+ "class_name": "email-message"+ },+ {+ "role": "Text",+ "depth": 2,+ "text": "Alice",+ "class_name": "sender"+ },+ {+ "role": "Group",+ "depth": 2,+ "text": "",+ "class_name": "message-body"+ },+ {+ "role": "Text",+ "depth": 3,+ "text": "The answer is Reply.\nPlease ship Friday."+ },+ {+ "role": "Button",+ "depth": 3,+ "text": ""+ },+ {+ "role": "Text",+ "depth": 4,+ "text": "Download attachment"+ },+ {+ "role": "Button",+ "depth": 3,+ "text": ""+ },+ {+ "role": "Text",+ "depth": 4,+ "text": "Forward message"+ }+ ],+ "checks": [+ {+ "path": "/1/actor",+ "expected": "Alice"+ },+ {+ "path": "/1/body",+ "expected": "The answer is Reply.\nPlease ship Friday."+ },+ {+ "path": "/1/parent_local_id",+ "expected": "thread"+ }+ ],+ "forbidden_context": [+ "Download attachment",+ "Forward message"+ ],+ "required_context": []+ }+]diff --git a/crates/screenpipe-semantic/evals/quality/mod.rs b/crates/screenpipe-semantic/evals/quality/mod.rsnew file mode 100644index 000000000..b9ee6d189--- /dev/null+++ b/crates/screenpipe-semantic/evals/quality/mod.rs@@ -0,0 +1,104 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use screenpipe_semantic::{+ adapt_captured_accessibility_tree, parsers::builtin_parser_registry, render_semantic_context,+ AppIdentity, CapturedAccessibilityNode, OutputBudget, ParseContext, TreeBudget,+ ValidatedParseOutcome,+};+use serde::{Deserialize, Serialize};+use serde_json::Value;+use tiktoken_rs::o200k_base_singleton;++#[derive(Deserialize)]+struct Case {+ id: String,+ app: AppIdentity,+ nodes: Vec<CapturedAccessibilityNode>,+ checks: Vec<Check>,+ forbidden_context: Vec<String>,+ #[serde(default)]+ required_context: Vec<String>,+}++#[derive(Deserialize)]+struct Check {+ path: String,+ expected: Value,+}++#[derive(Serialize)]+pub struct Report {+ pub id: String,+ pub checks: usize,+ pub passed: usize,+ pub failures: Vec<String>,+ pub raw_tokens: usize,+ pub context_tokens: usize,+ pub context: String,+}++/// Hand-authored synthetic gold, independent of parser output. Exact field+/// checks cover attribution, state and relationships, not just word presence.+pub fn evaluate() -> Result<Vec<Report>, Box<dyn std::error::Error>> {+ evaluate_cases(include_str!("cases.json"))+}++pub fn evaluate_cases(json: &str) -> Result<Vec<Report>, Box<dyn std::error::Error>> {+ let cases: Vec<Case> = serde_json::from_str(json)?;+ let registry = builtin_parser_registry()?;+ let tokenizer = o200k_base_singleton();+ let mut reports = Vec::new();+ for case in cases {+ let raw = serde_json::to_string(&case.nodes)?;+ let adapted = adapt_captured_accessibility_tree(&case.nodes, TreeBudget::default())?;+ let context = ParseContext {+ frame_id: 7,+ captured_at_unix_ms: 0,+ utc_offset_minutes: None,+ locale_hint: None,+ app: &case.app,+ input_content_hash: 1,+ };+ let parsed = registry.parse(&context, &adapted.tree, OutputBudget::default());+ let ValidatedParseOutcome::Handled(projection) = parsed.outcome else {+ return Err(+ format!("{} did not produce context: {:?}", case.id, parsed.outcome).into(),+ );+ };+ let rendered = render_semantic_context(&case.app, 7, &projection);+ let items = serde_json::to_value(projection.items())?;+ let mut failures = Vec::new();+ for check in &case.checks {+ if items.pointer(&check.path) != Some(&check.expected) {+ failures.push(format!(+ "{}: expected {}, got {:?}",+ check.path,+ check.expected,+ items.pointer(&check.path)+ ));+ }+ }+ for value in &case.forbidden_context {+ if rendered.contains(value) {+ failures.push(format!("unwanted context: {value}"));+ }+ }+ for value in &case.required_context {+ if !rendered.contains(value) {+ failures.push(format!("missing context: {value}"));+ }+ }+ let checks = case.checks.len() + case.forbidden_context.len() + case.required_context.len();+ reports.push(Report {+ id: case.id,+ checks,+ passed: checks - failures.len(),+ failures,+ raw_tokens: tokenizer.encode_ordinary(&raw).len(),+ context_tokens: tokenizer.encode_ordinary(&rendered).len(),+ context: rendered,+ });+ }+ Ok(reports)+}diff --git a/crates/screenpipe-semantic/evals/web_content/NATIVE_VALIDATION.md b/crates/screenpipe-semantic/evals/web_content/NATIVE_VALIDATION.mdnew file mode 100644index 000000000..4d7d231e4--- /dev/null+++ b/crates/screenpipe-semantic/evals/web_content/NATIVE_VALIDATION.md@@ -0,0 +1,114 @@+# Native capture validation++On September 21, 2026, an isolated AWS `mac-m4pro.metal` machine running macOS+15.7.9 (24G830), with SIP enabled, captured real accessibility trees through the+production `semantic_capture_probe`. Safari 26.6.1 visited public GitHub,+GitLab, Hacker News and Wikipedia pages. TextEdit 1.20 and Terminal displayed a+synthetic local document. No personal desktop or release machine was used.++The same seven private capture records were replayed at `d00a8e2deb` and+final parser commit `9f17d2a5f9` on a local Mac. The five browser surfaces previously abstained; the updated parsers+handle all five. Existing TextEdit and Terminal handling remains successful.+The native Mac capture code tested at `1ece5c01a6` remains unchanged by the Windows fixes.++| Native surface | Before | After | Context tokens | Known offscreen nodes excluded |+| --- | --- | --- | ---: | ---: |+| GitLab work item, top | Not handled | Title, author, visible description | 280 | 344 |+| Wikipedia article, top | Not handled | Title and visible article text | 355 | 1,613 |+| Wikipedia article, scrolled | Not handled | Visible section, no invented title | 722 | 1,564 |+| GitHub PR conversation, top | Not handled | Title, author, visible description | 164 | 373 |+| Hacker News discussion | Not handled | Story title and three attributed comments | 83 | 0 |+| TextEdit synthetic document | Handled | Handled, code indentation retained | 85 | 0 |+| Terminal synthetic output | Handled | Handled, code indentation retained | 279 | 0 |++There are **69/69 bounded checks** across these records for extracted facts,+item counts, attribution, authored text, code indentation, absent toolbar/sidebar+noise and parser failures. These are observations from seven surfaces, not a+population-wide accuracy estimate. See [sanitized metrics](native-macos-metrics.json).+Raw JSON token counts include structural and offscreen nodes; their reduction+measures representation size, not improvement over a previous semantic parser.++The live captures exposed and drove these corrections:++- WebKit omits inert DOM wrappers. Recognize bounded native structures for+ GitHub comment headers/end markers, GitLab description/sidebar boundaries,+ Hacker News sibling headers/bodies and Wikipedia article bodies.+- Preserve native proof that a positive-sized rectangle lies outside the+ window even when screenshot coordinate normalization discards its bounds.+ Zero-sized accessibility labels remain unknown visibility and fail open.+- Retain table column structure so duplicate row/column/header views are+ excluded without deduplicating repeated authored cells.+- Do not convert generic accessibility role descriptions into authored text;+ exclude recognized section-edit controls while retaining authored words.++Synthetic regressions model these observed structures in `tests/web_content.rs`+and `tests/capture_adapter.rs`. Raw probe output is deliberately not committed.++For new private captures, build the production probe on macOS or Windows, focus+an intended app, and run:++```sh+cargo run -p screenpipe-a11y --example semantic_capture_probe -- --delay-secs 5 > native.jsonl+cargo run -p screenpipe-semantic --example replay -- --jsonl < native.jsonl+# Explicit opt-in to inspect authored content; keep this output private too.+cargo run -p screenpipe-semantic --example replay -- --jsonl --include-context < native.jsonl+```++The AWS machine also ran the 162 semantic tests and the focused native geometry+and transient-serialization tests at `1ece5c01a6`. All passed. Local validation+included semantic library Clippy with warnings denied and the engine library+integration check.++## Windows++An isolated Azure Windows 11 VM (`Standard_DC8s_v3`, East US) used the prepared+Windows dev image and its own console session. Edge visited the same four public+pages. The first native capture at `d00a8e2deb` supplied AutomationId/ClassName+properties but discarded semantic containers/DOM fields: all four web parsers+abstained. The opt-in Windows path now retains that evidence and uses a bounded+RawView walk where Chromium's ControlView omits required content boundaries.+The ordinary capture path remains unchanged.++Native tests and fresh captures at **`9f17d2a5f9`** passed **169 semantic tests**,+**12 focused Windows a11y tests**, all four evals, semantic library Clippy and+diff checks. Independent replay passes **62/62 bounded checks** across the four+web surfaces, scrolled Wikipedia and a semantic-off control. See+[sanitized Windows metrics](native-windows-metrics.json).++| Native surface | Final result | Context tokens | Known offscreen nodes excluded |+| --- | --- | ---: | ---: |+| GitHub PR conversation, top | Title, local author and visible body | 130 | 56 |+| GitLab work item, top | Title, verified creator and description | 228 | 72 |+| Hacker News discussion | Title and three correctly attributed comments | 85 | 0 |+| Wikipedia article, top | Title and introductory article text | 206 | 0 |+| Wikipedia article, scrolled | Safely abstains | N/A | 209 |+| Edge semantic-off control | Legacy tree, no transient DOM/visibility fields | N/A | 7 |++These are separate live before/after captures, not a same-input token or speed+benchmark. Every semantic-on browser capture reached the unchanged 250 ms walk+budget. The scrolled ControlView control confirms that the viewport showed the+Efficiency/performance section, while RawView exhausted its budget before reaching+that verified body. The parser therefore abstains. It no longer emits the+incorrect offscreen title/introduction observed in the first Windows fix. This+is a correctness improvement, not complete deep-scroll coverage.++Native findings also drove synthetic regressions for combined semantic and+automation flags, finite positive off-window geometry versus unknown/zero+geometry, GitHub aggregate/link duplication, authored repeated paragraphs,+control-label exclusion, and complete bounded HN fragments with local identity+evidence. Six independent synthetic review checks improved from **2/6 to 6/6**;+these are deliberately separate from the live-capture checks. The original+partial-parent/link/suffix case remains supported, and code indentation is+preserved. The seven Mac records still pass **69/69** checks at this final parser+revision.++A separate final Notepad capture at `9f17d2a5f9` passed all 11 VM focus/content+assertions and 11 independent replay checks. The foreground HWND and process+were verified as Notepad, with no browser URL. The native walk completed in+112 ms without truncation; one document retained the synthetic marker, owner,+in-progress status, authored Copy/Reply words and code indentation in 76 context+tokens. These checks are separate from the 62 web/control checks above.++Raw captures, authored replay output and desktop recordings remain private.+Only synthetic regression fixtures and sanitized metrics are committed. Existing+release builders and other workflows were not modified.diff --git a/crates/screenpipe-semantic/evals/web_content/README.md b/crates/screenpipe-semantic/evals/web_content/README.mdnew file mode 100644index 000000000..c6df0463d--- /dev/null+++ b/crates/screenpipe-semantic/evals/web_content/README.md@@ -0,0 +1,88 @@+# New web app context coverage++Four new catalog profiles and dedicated parser registrations increase coverage+from 56 to 60 profiles and from 23 to 27 registered parsers. These are new apps,+not extra browser routes for already supported apps.++| App | Supported surface | Retained context |+|---|---|---|+| GitHub | github.com issue and pull-request conversation pages | Topic, source URL, description/comments, authors and attachment labels |+| GitLab | gitlab.com current issue/work-item detail pages | Topic, source URL, description and authored comments |+| Hacker News | `item?id=…` discussion pages | Story title, source URL, self-text and comments with authors |+| Wikipedia | Article paths under language subdomains | Heading, source URL and article text, including section headings |++Each parser requires its observed content markers. Missing markers, login screens,+drafts and previews abstain to generic accessibility; the shared family parsers+are deliberately not fallback candidates for these four profiles. Known-offscreen+text is suppressed by the existing capture adapter. A scrolled thread can retain+visible comments even when the title is no longer in the capture. Missing authors,+status and timestamps remain unknown. Comment order is not treated as reply+hierarchy: every post belongs to its observed discussion.++Source URLs omit fragments and queries except HN's required item id. Native post+ids distinguish equal text by different authors and deduplicate repeated wrappers.+Without a native post id, identity is explicitly ephemeral. Nested Markdown wrappers+do not duplicate paragraphs. Known control labels are removed only under controls;+links, attachment names, authored action words and code indentation remain.+A shared 32 KiB body budget and 64-post limit keep output within the default+projection budget. Truncation is explicit in context metadata.++## Evidence and limits++The following public page DOM structures were inspected in a task-owned hidden+browser on September 21, 2026. These links document the source of the structural+contracts, not a live native-capture accuracy measurement.++- [GitHub issue](https://github.com/laurent22/joplin/issues/10795):+ `react-issue-body`, `react-issue-comment`, nested `markdown-body`, `markdown-title`+ and author CSS-module prefixes.+- [GitHub pull request](https://github.com/screenpipe/screenpipe/pull/7167):+ `timeline-comment`, `js-comment-body`, `author`, `markdown-title`; preview bodies+ and composer content are excluded.+- [GitLab work item](https://gitlab.com/gitlab-org/gitlab/-/work_items/364818):+ `gl-heading-1`, `work-item-description`, `note-comment`, `note-text`,+ `author-name-link`. The old `/issues/` URL redirected to `/work_items/`.+- [Hacker News discussion](https://news.ycombinator.com/item?id=1):+ `fatitem`, `titleline`, `toptext`, `comtr`, `commtext`, `hnuser`.+- [Wikipedia article](https://en.wikipedia.org/wiki/Computer_programming):+ `firstHeading`, `mw-content-text`, `mw-parser-output`. Edit links, navigation+ boxes, sidebars and portals are excluded.++`cases.json` is hand-authored synthetic data using those shapes and invented+content. Tests replay through the capture adapter and real registry with macOS,+Windows and Linux identities/role variants. They do not prove that every native+browser walker supplies those DOM attributes. The subsequent isolated AWS Mac+and Azure Windows campaign exercised the production native walkers. See+[NATIVE_VALIDATION.md](NATIVE_VALIDATION.md) for observed results, fixes and limits.++Not covered: GitHub source/diff tabs, GitLab merge requests, self-hosted GitHub or+GitLab, HN listing pages, Wikipedia namespace pages or percent-encoded article+paths. Wikipedia's English main page abstains. Stack Overflow was not added:+its page presented a browser challenge, so its DOM contract was not verified.++## Reproduce++```sh+cargo test -p screenpipe-semantic --test web_content --locked+cargo run --release -p screenpipe-semantic --example web_content_eval --locked+```++The five fixtures have 60 exact field/relationship and exclusion assertions, all+passing. The same gold is checked across three platform identities; that does not+make these 180 independent facts. Regression tests additionally cover+routing, abstention, scrolled/unknown fields, stable identity, repeated wrappers,+bounded multibyte output and control reparenting.++| Fixture | Assertions | Raw JSON tokens | Context tokens |+|---|---:|---:|---:|+| GitHub issue | 15/15 | 413 | 74 |+| GitHub pull request | 9/9 | 267 | 44 |+| GitLab work item | 13/13 | 251 | 58 |+| Hacker News discussion | 15/15 | 409 | 76 |+| Wikipedia article | 8/8 | 215 | 45 |+| Total | **60/60** | **1,555** | **297** |++Exact `o200k_base` counts: context uses 80.9% fewer tokens than the serialized+accessibility fixture JSON. This is a representation comparison, not a reduction+against a previously supported semantic parser, and not a model-quality score.+The source URL is retained once per page because provenance is useful context.diff --git a/crates/screenpipe-semantic/evals/web_content/cases.json b/crates/screenpipe-semantic/evals/web_content/cases.jsonnew file mode 100644index 000000000..a96aa4508--- /dev/null+++ b/crates/screenpipe-semantic/evals/web_content/cases.json@@ -0,0 +1,725 @@+[+ {+ "id": "github_issue",+ "app": {+ "platform": "macos",+ "app_id": null,+ "executable": "Chrome",+ "display_name": "Browser",+ "version": null,+ "browser_url": "https://github.com/example/project/issues/42"+ },+ "nodes": [+ {+ "depth": 0,+ "role": "AXWebArea",+ "text": ""+ },+ {+ "depth": 1,+ "role": "AXStaticText",+ "text": "SIDEBAR_NOISE"+ },+ {+ "depth": 1,+ "role": "AXHeading",+ "text": "Keep code indentation",+ "class_name": "markdown-title"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "",+ "class_name": "react-issue-body",+ "dom_identifier": "issue-42"+ },+ {+ "depth": 2,+ "role": "AXLink",+ "text": "Alice",+ "class_name": "IssueBodyHeaderAuthor-module__authorLoginLink__abc"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "",+ "dom_identifier": "issue-body-viewer"+ },+ {+ "depth": 3,+ "role": "AXGroup",+ "text": "",+ "class_name": "markdown-body"+ },+ {+ "depth": 4,+ "role": "AXGroup",+ "text": "DUPLICATE_AGGREGATE",+ "class_name": "markdown-body"+ },+ {+ "depth": 5,+ "role": "AXStaticText",+ "text": "The parser must keep this example:"+ },+ {+ "depth": 5,+ "role": "AXStaticText",+ "text": "if ready:\n ship()\n record()"+ },+ {+ "depth": 5,+ "role": "AXButton",+ "text": "Copy"+ },+ {+ "depth": 6,+ "role": "AXStaticText",+ "text": "Copy"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "",+ "class_name": "react-issue-comment",+ "dom_identifier": "issuecomment-7"+ },+ {+ "depth": 2,+ "role": "AXLink",+ "text": "Bob",+ "class_name": "ActivityHeader-module__AuthorName__def"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "",+ "class_name": "markdown-body"+ },+ {+ "depth": 3,+ "role": "AXStaticText",+ "text": "Reply is an authored word here."+ },+ {+ "depth": 3,+ "role": "AXLink",+ "text": "Quoted Person",+ "class_name": "author"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "UNRELATED_COMMIT",+ "class_name": "markdown-body"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "UNSENT_PREVIEW",+ "class_name": "js-preview-body markdown-body"+ },+ {+ "depth": 1,+ "role": "AXTextArea",+ "text": "UNSENT_DRAFT"+ }+ ],+ "checks": [+ {+ "path": "/0/title",+ "expected": "Keep code indentation"+ },+ {+ "path": "/0/kind",+ "expected": "conversation"+ },+ {+ "path": "/1/body",+ "expected": "The parser must keep this example:\nif ready:\n ship()\n record()"+ },+ {+ "path": "/1/actor",+ "expected": "Alice"+ },+ {+ "path": "/1/parent_local_id",+ "expected": "page"+ },+ {+ "path": "/2/body",+ "expected": "Reply is an authored word here.\nQuoted Person"+ },+ {+ "path": "/2/actor",+ "expected": "Bob"+ },+ {+ "path": "/2/parent_local_id",+ "expected": "page"+ },+ {+ "path": "/0/metadata/url",+ "expected": "https://github.com/example/project/issues/42"+ }+ ],+ "forbidden_context": [+ "SIDEBAR_NOISE",+ "Copy",+ "DUPLICATE_AGGREGATE",+ "UNRELATED_COMMIT",+ "UNSENT_PREVIEW",+ "UNSENT_DRAFT"+ ]+ },+ {+ "id": "github_pull",+ "app": {+ "platform": "macos",+ "app_id": null,+ "executable": "Chrome",+ "display_name": "Browser",+ "version": null,+ "browser_url": "https://github.com/example/project/pull/51"+ },+ "nodes": [+ {+ "depth": 0,+ "role": "AXWebArea",+ "text": ""+ },+ {+ "depth": 1,+ "role": "AXStaticText",+ "text": "SIDEBAR_NOISE"+ },+ {+ "depth": 1,+ "role": "AXHeading",+ "text": "Preserve attachments",+ "class_name": "markdown-title"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "",+ "class_name": "timeline-comment",+ "dom_identifier": "pullrequest-51"+ },+ {+ "depth": 2,+ "role": "AXLink",+ "text": "Carol",+ "class_name": "author"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "UNSENT_PREVIEW",+ "class_name": "js-preview-body markdown-body"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "",+ "class_name": "comment-body markdown-body js-comment-body"+ },+ {+ "depth": 3,+ "role": "AXStaticText",+ "text": "Review this report."+ },+ {+ "depth": 3,+ "role": "AXLink",+ "text": "report.csv"+ },+ {+ "depth": 3,+ "role": "AXStaticText",+ "text": "OFFSCREEN_SECRET",+ "on_screen": false,+ "bounds": {+ "left": 0,+ "top": 4000,+ "width": 100,+ "height": 20+ }+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "",+ "class_name": "timeline-new-comment"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "UNSENT_PREVIEW",+ "class_name": "markdown-body js-preview-body"+ }+ ],+ "checks": [+ {+ "path": "/0/title",+ "expected": "Preserve attachments"+ },+ {+ "path": "/0/kind",+ "expected": "conversation"+ },+ {+ "path": "/1/body",+ "expected": "Review this report.\nreport.csv"+ },+ {+ "path": "/1/actor",+ "expected": "Carol"+ },+ {+ "path": "/1/parent_local_id",+ "expected": "page"+ },+ {+ "path": "/0/metadata/url",+ "expected": "https://github.com/example/project/pull/51"+ }+ ],+ "forbidden_context": [+ "SIDEBAR_NOISE",+ "UNSENT_PREVIEW",+ "OFFSCREEN_SECRET"+ ]+ },+ {+ "id": "gitlab",+ "app": {+ "platform": "macos",+ "app_id": null,+ "executable": "Chrome",+ "display_name": "Browser",+ "version": null,+ "browser_url": "https://gitlab.com/example/subgroup/project/-/work_items/42"+ },+ "nodes": [+ {+ "depth": 0,+ "role": "AXWebArea",+ "text": ""+ },+ {+ "depth": 1,+ "role": "AXStaticText",+ "text": "SIDEBAR_NOISE"+ },+ {+ "depth": 1,+ "role": "AXHeading",+ "text": "Recover the draft",+ "class_name": "gl-heading-1"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "",+ "class_name": "work-item-description description md"+ },+ {+ "depth": 2,+ "role": "AXStaticText",+ "text": "Expected: restore the saved description."+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "Read more",+ "class_name": "description-more"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "",+ "class_name": "note note-comment",+ "dom_identifier": "note_100"+ },+ {+ "depth": 2,+ "role": "AXLink",+ "text": "Dana",+ "class_name": "author-name-link"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "",+ "class_name": "note-text md"+ },+ {+ "depth": 3,+ "role": "AXStaticText",+ "text": "Confirmed on version 2."+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "",+ "class_name": "system-note"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "LABEL_EVENT_NOISE",+ "class_name": "note-text md"+ },+ {+ "depth": 1,+ "role": "Edit",+ "text": "UNSENT_DRAFT"+ }+ ],+ "checks": [+ {+ "path": "/0/title",+ "expected": "Recover the draft"+ },+ {+ "path": "/0/kind",+ "expected": "conversation"+ },+ {+ "path": "/1/body",+ "expected": "Expected: restore the saved description."+ },+ {+ "path": "/1/actor",+ "expected": null+ },+ {+ "path": "/1/parent_local_id",+ "expected": "page"+ },+ {+ "path": "/2/body",+ "expected": "Confirmed on version 2."+ },+ {+ "path": "/2/actor",+ "expected": "Dana"+ },+ {+ "path": "/2/parent_local_id",+ "expected": "page"+ },+ {+ "path": "/0/metadata/url",+ "expected": "https://gitlab.com/example/subgroup/project/-/work_items/42"+ }+ ],+ "forbidden_context": [+ "SIDEBAR_NOISE",+ "Read more",+ "LABEL_EVENT_NOISE",+ "UNSENT_DRAFT"+ ]+ },+ {+ "id": "hackernews",+ "app": {+ "platform": "macos",+ "app_id": null,+ "executable": "Chrome",+ "display_name": "Browser",+ "version": null,+ "browser_url": "https://news.ycombinator.com/item?id=42"+ },+ "nodes": [+ {+ "depth": 0,+ "role": "AXWebArea",+ "text": ""+ },+ {+ "depth": 1,+ "role": "AXStaticText",+ "text": "SIDEBAR_NOISE"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "",+ "class_name": "fatitem"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "",+ "class_name": "titleline"+ },+ {+ "depth": 3,+ "role": "AXLink",+ "text": "Local first software"+ },+ {+ "depth": 3,+ "role": "AXGroup",+ "text": "DOMAIN_NOISE",+ "class_name": "sitebit comhead"+ },+ {+ "depth": 2,+ "role": "AXLink",+ "text": "Erin",+ "class_name": "hnuser"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "How do you keep data available offline?",+ "class_name": "toptext"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "",+ "class_name": "comment-tree"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "",+ "class_name": "athing comtr",+ "dom_identifier": "101"+ },+ {+ "depth": 3,+ "role": "AXGroup",+ "text": "",+ "class_name": "comhead"+ },+ {+ "depth": 4,+ "role": "AXLink",+ "text": "Frank",+ "class_name": "hnuser"+ },+ {+ "depth": 3,+ "role": "AXGroup",+ "text": "",+ "class_name": "comment"+ },+ {+ "depth": 4,+ "role": "AXGroup",+ "text": "",+ "class_name": "commtext c00"+ },+ {+ "depth": 5,+ "role": "AXStaticText",+ "text": "Store an append-only log."+ },+ {+ "depth": 3,+ "role": "AXGroup",+ "text": "Reply",+ "class_name": "reply"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "",+ "class_name": "athing comtr",+ "dom_identifier": "102"+ },+ {+ "depth": 3,+ "role": "AXGroup",+ "text": "",+ "class_name": "comhead"+ },+ {+ "depth": 4,+ "role": "AXLink",+ "text": "Grace",+ "class_name": "hnuser"+ },+ {+ "depth": 3,+ "role": "AXGroup",+ "text": "",+ "class_name": "comment"+ },+ {+ "depth": 4,+ "role": "AXGroup",+ "text": "",+ "class_name": "commtext c00"+ },+ {+ "depth": 5,+ "role": "AXStaticText",+ "text": "Store an append-only log."+ }+ ],+ "checks": [+ {+ "path": "/0/title",+ "expected": "Local first software"+ },+ {+ "path": "/0/kind",+ "expected": "conversation"+ },+ {+ "path": "/1/body",+ "expected": "How do you keep data available offline?"+ },+ {+ "path": "/1/actor",+ "expected": "Erin"+ },+ {+ "path": "/1/parent_local_id",+ "expected": "page"+ },+ {+ "path": "/2/body",+ "expected": "Store an append-only log."+ },+ {+ "path": "/2/actor",+ "expected": "Frank"+ },+ {+ "path": "/2/parent_local_id",+ "expected": "page"+ },+ {+ "path": "/3/body",+ "expected": "Store an append-only log."+ },+ {+ "path": "/3/actor",+ "expected": "Grace"+ },+ {+ "path": "/3/parent_local_id",+ "expected": "page"+ },+ {+ "path": "/0/metadata/url",+ "expected": "https://news.ycombinator.com/item?id=42"+ }+ ],+ "forbidden_context": [+ "SIDEBAR_NOISE",+ "DOMAIN_NOISE",+ "Reply"+ ]+ },+ {+ "id": "wikipedia",+ "app": {+ "platform": "macos",+ "app_id": null,+ "executable": "Chrome",+ "display_name": "Browser",+ "version": null,+ "browser_url": "https://en.wikipedia.org/wiki/Computer_programming"+ },+ "nodes": [+ {+ "depth": 0,+ "role": "AXWebArea",+ "text": ""+ },+ {+ "depth": 1,+ "role": "AXStaticText",+ "text": "SIDEBAR_NOISE"+ },+ {+ "depth": 1,+ "role": "AXHeading",+ "text": "Computer programming",+ "dom_identifier": "firstHeading"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "",+ "dom_identifier": "mw-content-text"+ },+ {+ "depth": 2,+ "role": "AXGroup",+ "text": "",+ "class_name": "mw-parser-output"+ },+ {+ "depth": 3,+ "role": "AXStaticText",+ "text": "Programming constructs executable instructions."+ },+ {+ "depth": 3,+ "role": "AXHeading",+ "text": "History"+ },+ {+ "depth": 3,+ "role": "AXGroup",+ "text": "",+ "class_name": "mw-editsection"+ },+ {+ "depth": 4,+ "role": "AXLink",+ "text": "EDIT_ACTION"+ },+ {+ "depth": 3,+ "role": "AXStaticText",+ "text": "Programs can be written in many languages."+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "CATEGORY_NOISE",+ "dom_identifier": "catlinks"+ },+ {+ "depth": 1,+ "role": "AXGroup",+ "text": "UNRELATED_PREVIEW",+ "class_name": "mw-parser-output"+ }+ ],+ "checks": [+ {+ "path": "/0/title",+ "expected": "Computer programming"+ },+ {+ "path": "/0/kind",+ "expected": "document"+ },+ {+ "path": "/0/body",+ "expected": "Programming constructs executable instructions.\nHistory\nPrograms can be written in many languages."+ },+ {+ "path": "/0/metadata/url",+ "expected": "https://en.wikipedia.org/wiki/Computer_programming"+ }+ ],+ "forbidden_context": [+ "SIDEBAR_NOISE",+ "EDIT_ACTION",+ "CATEGORY_NOISE",+ "UNRELATED_PREVIEW"+ ]+ }+]diff --git a/crates/screenpipe-semantic/evals/web_content/native-macos-metrics.json b/crates/screenpipe-semantic/evals/web_content/native-macos-metrics.jsonnew file mode 100644index 000000000..97cbb3bc6--- /dev/null+++ b/crates/screenpipe-semantic/evals/web_content/native-macos-metrics.json@@ -0,0 +1,94 @@+{+ "capture_platform": "macOS 15.7.9 arm64, AWS mac-m4pro.metal",+ "browser": "Safari 26.6.1",+ "before_commit": "d00a8e2deb98a3d5af2777b23eadd160279e5f1c",+ "after_commit": "9f17d2a5f9d7b9c205b7db7ef98d19fc8b29a7eb",+ "method": "Seven real production AX captures from the isolated AWS Mac, replayed at d00a8e2deb and final parser commit 9f17d2a5f9 on the local Mac. Native Mac capture code was unchanged by the Windows fixes. Raw content is private. These bounded checks are not a population-wide accuracy estimate; raw JSON counts measure representation size and include offscreen nodes.",+ "results": [+ {+ "surface": "GitLab issue, top",+ "before_outcome": "not_handled",+ "after_outcome": "handled",+ "items": 2,+ "raw_json_tokens": 30024,+ "context_tokens": 280,+ "known_offscreen_nodes": 344,+ "checks": 11,+ "passed": 11,+ "failures": []+ },+ {+ "surface": "Wikipedia article, top",+ "before_outcome": "not_handled",+ "after_outcome": "handled",+ "items": 1,+ "raw_json_tokens": 94324,+ "context_tokens": 355,+ "known_offscreen_nodes": 1613,+ "checks": 10,+ "passed": 10,+ "failures": []+ },+ {+ "surface": "Wikipedia article, scrolled",+ "before_outcome": "not_handled",+ "after_outcome": "handled",+ "items": 1,+ "raw_json_tokens": 90816,+ "context_tokens": 722,+ "known_offscreen_nodes": 1564,+ "checks": 10,+ "passed": 10,+ "failures": []+ },+ {+ "surface": "GitHub pull-request conversation, top",+ "before_outcome": "not_handled",+ "after_outcome": "handled",+ "items": 2,+ "raw_json_tokens": 32103,+ "context_tokens": 164,+ "known_offscreen_nodes": 373,+ "checks": 9,+ "passed": 9,+ "failures": []+ },+ {+ "surface": "Hacker News discussion",+ "before_outcome": "not_handled",+ "after_outcome": "handled",+ "items": 4,+ "raw_json_tokens": 8394,+ "context_tokens": 83,+ "known_offscreen_nodes": 0,+ "checks": 11,+ "passed": 11,+ "failures": []+ },+ {+ "surface": "TextEdit synthetic document",+ "before_outcome": "handled",+ "after_outcome": "handled",+ "items": 1,+ "raw_json_tokens": 463,+ "context_tokens": 85,+ "known_offscreen_nodes": 0,+ "checks": 10,+ "passed": 10,+ "failures": []+ },+ {+ "surface": "Terminal synthetic output",+ "before_outcome": "handled",+ "after_outcome": "handled",+ "items": 1,+ "raw_json_tokens": 1170,+ "context_tokens": 279,+ "known_offscreen_nodes": 0,+ "checks": 8,+ "passed": 8,+ "failures": []+ }+ ],+ "capture_commit": "1ece5c01a6fee1114242464f9f16da7c466c491b"+}diff --git a/crates/screenpipe-semantic/evals/web_content/native-windows-metrics.json b/crates/screenpipe-semantic/evals/web_content/native-windows-metrics.jsonnew file mode 100644index 000000000..749543335--- /dev/null+++ b/crates/screenpipe-semantic/evals/web_content/native-windows-metrics.json@@ -0,0 +1,142 @@+{+ "capture_platform": "Windows 11, isolated Azure Standard_DC8s_v3 VM",+ "browser": "153.0.4234.48",+ "capture_and_native_test_commit": "9f17d2a5f9d7b9c205b7db7ef98d19fc8b29a7eb",+ "baseline_capture_commit": "d00a8e2deb98a3d5af2777b23eadd160279e5f1c",+ "previous_windows_fix_commit": "242579f39f0f38964e2ca389f084ce64edc82f40",+ "method": "Fresh production UIA captures replayed independently through the final parser. Before/after captures are separate observations, not a same-input speed or token benchmark. Checks apply only to these captured surfaces. Raw content is private.",+ "limitation": "All semantic-on browser captures exhausted the unchanged 250ms walk budget. Deep scrolled Wikipedia safely abstains; full-page and universal viewport coverage are not claimed.",+ "results": [+ {+ "surface": "GitHub PR, top",+ "before_outcome": "not_handled",+ "after_outcome": "handled",+ "items": 2,+ "context_tokens": 130,+ "known_offscreen_nodes": 56,+ "capture": {+ "node_count": 428,+ "truncated": true,+ "truncation_reason": "Timeout",+ "walk_duration_ms": 252+ },+ "checks": 12,+ "passed": 12,+ "failures": []+ },+ {+ "surface": "GitLab work item, top",+ "before_outcome": "not_handled",+ "after_outcome": "handled",+ "items": 2,+ "context_tokens": 228,+ "known_offscreen_nodes": 72,+ "capture": {+ "node_count": 428,+ "truncated": true,+ "truncation_reason": "Timeout",+ "walk_duration_ms": 252+ },+ "checks": 12,+ "passed": 12,+ "failures": []+ },+ {+ "surface": "Hacker News discussion",+ "before_outcome": "not_handled",+ "after_outcome": "handled",+ "items": 4,+ "context_tokens": 85,+ "known_offscreen_nodes": 0,+ "capture": {+ "node_count": 424,+ "truncated": true,+ "truncation_reason": "Timeout",+ "walk_duration_ms": 251+ },+ "checks": 12,+ "passed": 12,+ "failures": []+ },+ {+ "surface": "Wikipedia article, top",+ "before_outcome": "not_handled",+ "after_outcome": "handled",+ "items": 1,+ "context_tokens": 206,+ "known_offscreen_nodes": 0,+ "capture": {+ "node_count": 423,+ "truncated": true,+ "truncation_reason": "Timeout",+ "walk_duration_ms": 251+ },+ "checks": 12,+ "passed": 12,+ "failures": []+ },+ {+ "surface": "Wikipedia article, scrolled",+ "before_outcome": "handled with stale offscreen introduction",+ "after_outcome": "not_handled",+ "items": 0,+ "context_tokens": null,+ "known_offscreen_nodes": 209,+ "capture": {+ "node_count": 431,+ "truncated": true,+ "truncation_reason": "Timeout",+ "walk_duration_ms": 252+ },+ "checks": 7,+ "passed": 7,+ "failures": []+ },+ {+ "surface": "Edge semantic-off control",+ "before_outcome": "legacy control",+ "after_outcome": "not_handled",+ "items": 0,+ "context_tokens": null,+ "known_offscreen_nodes": 7,+ "capture": {+ "node_count": 1769,+ "truncated": false,+ "truncation_reason": "None",+ "walk_duration_ms": 217+ },+ "checks": 7,+ "passed": 7,+ "failures": []+ }+ ],+ "notepad_acceptance": {+ "tested_commit": "9f17d2a5f9d7b9c205b7db7ef98d19fc8b29a7eb",+ "surface": "Notepad synthetic document",+ "capture": {+ "node_count": 48,+ "truncated": false,+ "truncation_reason": "None",+ "walk_duration_ms": 112+ },+ "outcome": "handled",+ "items": 1,+ "context_tokens": 76,+ "independent_replay_checks": {+ "notepad_identity": true,+ "no_browser_url": true,+ "untruncated": true,+ "handled": true,+ "one_item": true,+ "no_failures": true,+ "marker": true,+ "owner": true,+ "status": true,+ "authored_actions": true,+ "indentation": true+ },+ "vm_focus_and_content_assertions": 11,+ "vm_assertions_passed": 11,+ "independent_checks_passed": 11+ }+}diff --git a/crates/screenpipe-semantic/examples/context_quality.rs b/crates/screenpipe-semantic/examples/context_quality.rsnew file mode 100644index 000000000..72e453123--- /dev/null+++ b/crates/screenpipe-semantic/examples/context_quality.rs@@ -0,0 +1,10 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++#[path = "../evals/quality/mod.rs"]+mod quality;++fn main() -> Result<(), Box<dyn std::error::Error>> {+ println!("{}", serde_json::to_string_pretty(&quality::evaluate()?)?);+ Ok(())+}diff --git a/crates/screenpipe-semantic/examples/replay.rs b/crates/screenpipe-semantic/examples/replay.rsindex a2ebc2b14..56dc17422 100644--- a/crates/screenpipe-semantic/examples/replay.rs+++ b/crates/screenpipe-semantic/examples/replay.rs@@ -50,6 +50,9 @@ struct ReplayMetrics {output_text_bytes: usize,raw_json_bytes: usize,raw_json_tokens: usize,+ /// Authored content is emitted only with an explicit diagnostic opt-in.+ #[serde(skip_serializing_if = "Option::is_none")]+ semantic_context: Option<String>,semantic_context_bytes: Option<usize>,semantic_context_tokens: Option<usize>,semantic_token_reduction_percent: Option<f64>,@@ -61,7 +64,10 @@ struct ReplayMetrics {fn main() -> Result<(), Box<dyn std::error::Error>> {let args: Vec<String> = std::env::args().skip(1).collect();if args.as_slice() == ["--jsonl"] {- return replay_jsonl();+ return replay_jsonl(false);+ }+ if args.as_slice() == ["--jsonl", "--include-context"] {+ return replay_jsonl(true);}let (app, content_hash) = parse_args(args)?;let mut input = String::new();@@ -77,12 +83,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {}let nodes: Vec<CapturedAccessibilityNode> = serde_json::from_str(&input)?;let registry = builtin_parser_registry()?;- let metrics = replay_one(®istry, 0, 0, content_hash, app, nodes)?;+ let metrics = replay_one(®istry, 0, 0, content_hash, app, nodes, false)?;println!("{}", serde_json::to_string_pretty(&metrics)?);Ok(())}-fn replay_jsonl() -> Result<(), Box<dyn std::error::Error>> {+fn replay_jsonl(include_context: bool) -> Result<(), Box<dyn std::error::Error>> {let registry = builtin_parser_registry()?;let stdin = std::io::stdin();let mut records = 0usize;@@ -117,6 +123,7 @@ fn replay_jsonl() -> Result<(), Box<dyn std::error::Error>> {input.content_hash,input.app,input.nodes,+ include_context,)?;println!("{}", serde_json::to_string(&metrics)?);}@@ -130,6 +137,7 @@ fn replay_one(content_hash: u64,app: AppIdentity,nodes: Vec<CapturedAccessibilityNode>,+ include_context: bool,) -> Result<ReplayMetrics, Box<dyn std::error::Error>> {let raw_json = serde_json::to_string(&nodes)?;let raw_json_bytes = raw_json.len();@@ -198,6 +206,7 @@ fn replay_one(output_text_bytes,raw_json_bytes,raw_json_tokens,+ semantic_context: include_context.then_some(semantic_context).flatten(),semantic_context_bytes,semantic_context_tokens,semantic_token_reduction_percent,diff --git a/crates/screenpipe-semantic/examples/web_content_eval.rs b/crates/screenpipe-semantic/examples/web_content_eval.rsnew file mode 100644index 000000000..10abc855b--- /dev/null+++ b/crates/screenpipe-semantic/examples/web_content_eval.rs@@ -0,0 +1,16 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++#[allow(dead_code)]+#[path = "../evals/quality/mod.rs"]+mod quality;++fn main() -> Result<(), Box<dyn std::error::Error>> {+ println!(+ "{}",+ serde_json::to_string_pretty(&quality::evaluate_cases(include_str!(+ "../evals/web_content/cases.json"+ ),)?)?+ );+ Ok(())+}diff --git a/crates/screenpipe-semantic/src/capture.rs b/crates/screenpipe-semantic/src/capture.rsindex 4fc2683e3..41f87266c 100644--- a/crates/screenpipe-semantic/src/capture.rs+++ b/crates/screenpipe-semantic/src/capture.rs@@ -25,6 +25,10 @@ pub struct CapturedAccessibilityNode {pub bounds: Option<NodeBounds>,#[serde(default, skip_serializing_if = "Option::is_none")]pub on_screen: Option<bool>,+ /// Native geometry proved this node is outside the window before bounds+ /// normalization discarded it. Absent in legacy captures.+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]+ pub offscreen_geometry: bool,#[serde(default, skip_serializing_if = "Option::is_none")]pub automation_id: Option<String>,#[serde(default, skip_serializing_if = "Option::is_none")]@@ -108,7 +112,9 @@ pub struct AdaptedSemanticTree {/// (`aria-live` regions, visually-hidden labels) land here and describe what/// IS on screen, so treating them as scrollback deleted visible content. On/// real Windows captures that cost ~85% of Claude desktop conversation-/// extraction. No geometry is therefore unknown visibility, and fails open.+/// extraction. No geometry is therefore unknown visibility, and fails open,+/// unless the native walker retained explicit `offscreen_geometry` evidence+/// before screenshot normalization discarded an off-window rectangle.pub fn adapt_captured_accessibility_tree(nodes: &[CapturedAccessibilityNode],budget: TreeBudget,@@ -162,7 +168,8 @@ pub fn adapt_captured_accessibility_tree(class_count += 1;}let classes = &class_buffer[..class_count];- let known_offscreen = node.on_screen == Some(false) && node.bounds.is_some();+ let known_offscreen =+ node.on_screen == Some(false) && (node.bounds.is_some() || node.offscreen_geometry);let source_description = node.help_text.as_deref()diff --git a/crates/screenpipe-semantic/src/parsers/catalog.rs b/crates/screenpipe-semantic/src/parsers/catalog.rsindex 995fb9445..63ef175c7 100644--- a/crates/screenpipe-semantic/src/parsers/catalog.rs+++ b/crates/screenpipe-semantic/src/parsers/catalog.rs@@ -6,6 +6,19 @@ use crate::{AccessibilityAttribute, AppIdentity, AppVersionRequirement, ParserManifest, ParserScope,Platform, SemanticKind,};+use regex::RegexSet;+use std::sync::LazyLock;++// Use the same anchored patterns as registry routing. URL marker substrings+// are only a cheap prefilter, never proof of an app's origin or document path.+static PROFILE_URL_PATTERNS: LazyLock<Vec<RegexSet>> = LazyLock::new(|| {+ BUILTIN_APP_PROFILES+ .iter()+ .map(|profile| {+ RegexSet::new(profile.url_patterns).expect("built-in profile URL patterns must compile")+ })+ .collect()+});#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum AppFamily {@@ -20,7 +33,7 @@ pub enum AppFamily {/// Public app identity and family membership for one built-in parser profile.///-/// The catalog describes 56 supported app targets using public app identities,+/// The catalog describes 60 supported app targets using public app identities,/// URL patterns, and stable accessibility contracts.#[derive(Debug, Clone, Copy)]pub struct BuiltinAppProfile {@@ -243,6 +256,26 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&[],None,),+ profile(+ "github",+ "GitHub",+ &[C],+ &[],+ &[],+ &[r"^https://github\.com/[^/?#]+/[^/?#]+/(?:issues|pull)/[0-9]+(?:[?#].*)?$"],+ &["github.com/"],+ Some("https://github.com/screenpipe/screenpipe/issues/1"),+ ),+ profile(+ "gitlab",+ "GitLab",+ &[C],+ &[],+ &[],+ &[r"^https://gitlab\.com/(?:[^/?#]+/)+-/(?:issues|work_items)/[0-9]+(?:[?#].*)?$"],+ &["gitlab.com/"],+ Some("https://gitlab.com/gitlab-org/gitlab/-/work_items/1"),+ ),profile("gmail","Gmail",@@ -253,6 +286,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&["mail.google.com/"],Some("https://mail.google.com/mail/u/0/#inbox/example"),),+ profile(+ "hackernews",+ "Hacker News",+ &[C],+ &[],+ &[],+ &[r"^https://news\.ycombinator\.com/item\?id=[0-9]+(?:[&#].*)?$"],+ &["news.ycombinator.com/"],+ Some("https://news.ycombinator.com/item?id=1"),+ ),profile("hey","HEY",@@ -331,9 +374,9 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&[C],&["com.microsoft.teams2"],&["Microsoft Teams", "ms-teams.exe", "teams"],- &[],- &[],- None,+ &[r"^https://teams\.microsoft\.com/"],+ &["teams.microsoft.com/"],+ Some("https://teams.microsoft.com/v2/"),),profile("microsofttodo",@@ -341,9 +384,9 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&[T],&["com.microsoft.to-do-mac"],&["Microsoft To Do", "Todo.exe", "todo"],- &[],- &[],- None,+ &[r"^https://to-do\.office\.com/"],+ &["to-do.office.com/"],+ Some("https://to-do.office.com/tasks/"),),profile("microsoftword",@@ -362,11 +405,11 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&[],&[],&[- r"^https?://[^/]*officeapps\.live\.com/we/wordeditorframe",- r"^https?://[^/]*onedrive\.live\.com/[^?#]+/_layouts/15/[Dd]oc\.aspx",- r"^https?://[^/]*sharepoint\.com/[^?#]+/_layouts/15/[Dd]oc\.aspx",+ r"^https?://(?:[a-zA-Z0-9-]+\.)*officeapps\.live\.com/we/wordeditorframe",+ r"^https?://(?:[a-zA-Z0-9-]+\.)*onedrive\.live\.com/[^?#]+/_layouts/15/[Dd]oc\.aspx",+ r"^https?://(?:[a-zA-Z0-9-]+\.)*sharepoint\.com/[^?#]+/_layouts/15/[Dd]oc\.aspx",r"^https?://word\.cloud\.microsoft/",- r"^https?://[^/]*office\.com/[^?#]+\.docx",+ r"^https?://(?:[a-zA-Z0-9-]+\.)*office\.com/[^?#]+\.docx",],&["officeapps.live.com/",@@ -403,9 +446,13 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&[D],&["notion.id"],&["Notion", "Notion.exe", "notion"],- &[],- &[],- None,+ &[+ r"^https://(?:www\.)?notion\.so/",+ r"^https://app\.notion\.com/",+ r"^https://[a-zA-Z0-9-]+\.notion\.site/",+ ],+ &["notion.so/", "app.notion.com/", ".notion.site/"],+ Some("https://www.notion.so/workspace/example"),),profile("obsidian",@@ -453,9 +500,9 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&[C],&["com.tinyspeck.slackmacgap"],&["Slack", "slack.exe", "slack"],- &[],- &[],- None,+ &[r"^https://app\.slack\.com/client/"],+ &["app.slack.com/client/"],+ Some("https://app.slack.com/client/TEXAMPLE/CEXAMPLE"),),profile("sparkdesktop",@@ -550,9 +597,12 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&[E],&["com.microsoft.VSCode"],&["Visual Studio Code", "Code.exe", "code"],- &[],- &[],- None,+ &[+ r"^https://(?:insiders\.)?vscode\.dev/",+ r"^https://github\.dev/",+ ],+ &["vscode.dev/", "github.dev/"],+ Some("https://vscode.dev/"),),profile("warp",@@ -584,6 +634,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&["web.whatsapp.com/"],Some("https://web.whatsapp.com/"),),+ profile(+ "wikipedia",+ "Wikipedia",+ &[D],+ &[],+ &[],+ &[r"^https://[a-z][a-z0-9-]*\.wikipedia\.org/wiki/[^:%?#]+(?:[?#].*)?$"],+ &[".wikipedia.org/wiki/"],+ Some("https://en.wikipedia.org/wiki/Computer_programming"),+ ),profile("windowsconsole","Console",@@ -674,18 +734,30 @@ pub(crate) fn profiles_for_family() -> impl Iterator<Item = &'static BuiltinAppProfile> {BUILTIN_APP_PROFILES.iter()- .filter(move |profile| profile.families.contains(&family))+ // These reading surfaces have dedicated contracts. In particular, a+ // Markdown draft must not fall through to the document/editor family.+ .filter(move |profile| {+ profile.families.contains(&family) && !super::web_content::IDS.contains(&profile.id)+ })}pub(crate) fn profile_for(family: AppFamily,app: &AppIdentity,) -> Option<&'static BuiltinAppProfile> {- profiles_for_family(family).find(|profile| profile.matches(app))+ BUILTIN_APP_PROFILES+ .iter()+ .enumerate()+ .find(|(index, profile)| {+ profile.families.contains(&family)+ && !super::web_content::IDS.contains(&profile.id)+ && profile.matches(app, *index)+ })+ .map(|(_, profile)| profile)}impl BuiltinAppProfile {- fn matches(&self, app: &AppIdentity) -> bool {+ fn matches(&self, app: &AppIdentity, index: usize) -> bool {let app_id_match = app.app_id.as_deref().is_some_and(|candidate| {self.app_ids.iter()@@ -697,12 +769,15 @@ impl BuiltinAppProfile {.iter().any(|expected| expected.eq_ignore_ascii_case(basename))});- let url_match = app.browser_url.as_deref().is_some_and(|url| {+ if app_id_match || executable_match {+ return true;+ }+ app.browser_url.as_deref().is_some_and(|url| {self.url_markers.iter().any(|marker| contains_ascii_case_insensitive(url, marker))- });- app_id_match || executable_match || url_match+ && PROFILE_URL_PATTERNS[index].is_match(url)+ })}}@@ -716,7 +791,7 @@ pub(crate) fn manifest_for_family(let profiles: Vec<_> = profiles_for_family(family).collect();ParserManifest {id: id.into(),- parser_version: "1".into(),+ parser_version: "2".into(),schema_version: 1,scope: ParserScope::Family,platforms: vec![Platform::Macos, Platform::Windows, Platform::Linux],@@ -743,6 +818,9 @@ pub(crate) fn manifest_for_family(}pub(crate) fn contains_ascii_case_insensitive(value: &str, needle: &str) -> bool {+ if needle.is_empty() {+ return true;+ }value.as_bytes().windows(needle.len())diff --git a/crates/screenpipe-semantic/src/parsers/editor.rs b/crates/screenpipe-semantic/src/parsers/editor.rsindex 296d8a7c1..40aafd857 100644--- a/crates/screenpipe-semantic/src/parsers/editor.rs+++ b/crates/screenpipe-semantic/src/parsers/editor.rs@@ -41,7 +41,7 @@ impl EditorFamilyParser {],60,);- manifest.parser_version = "2".into();+ manifest.parser_version = "3".into();Self { manifest }}}diff --git a/crates/screenpipe-semantic/src/parsers/families.rs b/crates/screenpipe-semantic/src/parsers/families.rsindex cae020039..9abb35e4c 100644--- a/crates/screenpipe-semantic/src/parsers/families.rs+++ b/crates/screenpipe-semantic/src/parsers/families.rs@@ -90,8 +90,10 @@ impl FamilyParser {priority,);if family == AppFamily::Conversation {- manifest.parser_version = "4".into();+ manifest.parser_version = "6".into();} else if family == AppFamily::Mail {+ manifest.parser_version = "5".into();+ } else if family == AppFamily::Task {manifest.parser_version = "3".into();}Self { family, manifest }@@ -720,9 +722,13 @@ fn parse_tasks(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<Semantic);let mut items = Vec::new();for (index, node) in candidates.into_iter().enumerate() {- let title = first_marked_text_in(tree, node, &["task-title", "item-title", "title"])- .or_else(|| first_text_in(tree, node));- let Some(title) = title.map(str::trim).filter(|title| !title.is_empty()) else {+ let title = task_field(tree, node, &["task-title", "item-title", "title"])+ .or_else(|| first_text_in(tree, node).map(str::to_owned));+ let Some(title) = title+ .as_deref()+ .map(str::trim)+ .filter(|title| !title.is_empty())+ else {continue;};let mut task = SemanticItem::new(@@ -732,12 +738,66 @@ fn parse_tasks(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticIdentityQuality::Derived,);task.title = Some(title.to_owned());- task.status =- if signature_has_any(tree, node, &["completed", "complete", "done", "checked"]) {- Some("completed".into())- } else {- first_marked_text_in(tree, node, &["status"]).map(str::to_owned)- };+ // A visible status field outranks a wrapper's presentation class.+ // Substring matching made `unchecked` and `incomplete` completed, and+ // also mistook human-readable task descriptions for state evidence.+ task.status = task_field(+ tree,+ node,+ &["status", "task-status", "item-status", "issue-status"],+ )+ .or_else(|| {+ structural_marker_is(+ tree,+ node,+ &[+ "completed",+ "complete",+ "done",+ "checked",+ "is-completed",+ "is-complete",+ "is-done",+ "is-checked",+ "task-completed",+ ],+ )+ .then(|| "completed".to_owned())+ });+ for (key, markers) in [+ (+ "assignee",+ &[+ "assignee",+ "task-assignee",+ "item-assignee",+ "issue-assignee",+ ][..],+ ),+ (+ "due",+ &[+ "due-date",+ "task-due-date",+ "task-due",+ "item-due-date",+ "issue-due-date",+ ][..],+ ),+ (+ "priority",+ &[+ "priority",+ "task-priority",+ "item-priority",+ "issue-priority",+ ][..],+ ),+ ] {+ if let Some(value) = task_field(tree, node, markers) {+ task.metadata.insert(key.into(), value);+ }+ }task.metadata.insert("app".into(), profile.display_name.into());task.metadata.insert("family".into(), "task".into());@@ -747,6 +807,35 @@ fn parse_tasks(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<Semanticitems}+fn structural_marker_is(tree: &SemanticTree, node: NodeId, markers: &[&str]) -> bool {+ [tree.identifier(node), tree.dom_identifier(node)]+ .into_iter()+ .flatten()+ .chain(tree.classes(node))+ .any(|field| {+ markers+ .iter()+ .any(|marker| field.eq_ignore_ascii_case(marker))+ })+}++/// Retain explicit decision fields, including labels inside structural wrappers.+/// Do not infer owners or deadlines from arbitrary task prose or action labels.+fn task_field(tree: &SemanticTree, root: NodeId, markers: &[&str]) -> Option<String> {+ tree.descendants(root)+ .filter(|node| structural_marker_is(tree, *node, markers))+ .find_map(|node| {+ let value = collect_text(tree, node).or_else(|| {+ node_content(tree, node)+ .filter(|value| !is_control_type_label(value, tree.role(node)))+ .map(str::to_owned)+ })?;+ let value = value.trim();+ (!value.is_empty() && value.len() <= 240 && !value.contains(['\n', '\r']))+ .then(|| value.to_owned())+ })+}+fn parse_calendar(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticItem> {let candidates = leaf_marker_nodes(tree,@@ -1370,7 +1459,7 @@ fn is_simple_slash_date(token: &str) -> bool {fn tree_any(tree: &SemanticTree, predicate: impl Fn(NodeId) -> bool) -> bool {tree.roots()- .any(|root| tree.descendants(root).any(|node| predicate(node)))+ .any(|root| tree.descendants(root).any(&predicate))}/// `collect_text` starting below `root`. A Fluent message container is itself a@@ -1684,7 +1773,7 @@ fn collect_text(tree: &SemanticTree, root: NodeId) -> Option<String> {let mut lines: Vec<&str> = Vec::new();let mut bytes = 0usize;'nodes: for node in tree.descendants(root) {- if !is_text_role(tree.role(node)) {+ if !is_text_role(tree.role(node)) || inside_action_control(tree, node, root) {continue;}let Some(content) = node_content(tree, node) else {@@ -1705,6 +1794,47 @@ fn collect_text(tree: &SemanticTree, root: NodeId) -> Option<String> {(!lines.is_empty()).then(|| lines.join("\n"))}+fn inside_action_control(tree: &SemanticTree, node: NodeId, root: NodeId) -> bool {+ // Captured trees can reparent real body text below an interactive node.+ // A control ancestor alone is insufficient: discard only a recognized+ // action label, preserving link previews, poll choices and attachment names.+ let Some(content) = node_content(tree, node) else {+ return false;+ };+ if ![+ "Reply",+ "Reply to thread",+ "Add reaction",+ "More actions",+ "Copy message",+ "Copy link",+ "Edit message",+ "Delete message",+ "Download attachment",+ "Forward message",+ ]+ .iter()+ .any(|label| content.eq_ignore_ascii_case(label))+ {+ return false;+ }+ let mut ancestor = tree.parent(node);+ while let Some(parent) = ancestor {+ if parent == root {+ return false;+ }+ if tree.role(parent).is_some_and(|role| {+ ["Button", "AXButton", "MenuItem", "AXMenuItem"]+ .iter()+ .any(|control| role.eq_ignore_ascii_case(control))+ }) {+ return true;+ }+ ancestor = tree.parent(parent);+ }+ false+}+fn truncate_body(value: &str) -> &str {if value.len() <= MAX_COLLECTED_BODY_BYTES {return value;diff --git a/crates/screenpipe-semantic/src/parsers/mod.rs b/crates/screenpipe-semantic/src/parsers/mod.rsindex 1c22a5bd7..fcad27041 100644--- a/crates/screenpipe-semantic/src/parsers/mod.rs+++ b/crates/screenpipe-semantic/src/parsers/mod.rs@@ -10,6 +10,7 @@ mod families;mod native_macos;mod obsidian;mod persisted_web;+mod web_content;pub use catalog::{builtin_app_profiles, AppFamily, BuiltinAppProfile};pub use chatgpt::ChatGptParser;@@ -22,6 +23,9 @@ use crate::{ParserRegistry, RegistryError};use native_macos::native_macos_parsers;pub fn register_builtin_parsers(registry: &mut ParserRegistry) -> Result<(), RegistryError> {+ for &app in web_content::IDS {+ registry.register(Box::new(web_content::WebContentParser::new(app)))?;+ }registry.register(Box::new(ChatGptParser::new()))?;registry.register(Box::new(ClaudeParser::new()))?;registry.register(Box::new(ObsidianParser::new()))?;diff --git a/crates/screenpipe-semantic/src/parsers/web_content.rs b/crates/screenpipe-semantic/src/parsers/web_content.rsnew file mode 100644index 000000000..2123f8531--- /dev/null+++ b/crates/screenpipe-semantic/src/parsers/web_content.rs@@ -0,0 +1,917 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use super::catalog::builtin_app_profiles;+use crate::{+ AccessibilityAttribute, AppVersionRequirement, IdentityQuality, NodeId, ParseContext,+ ParseOutcome, ParserManifest, ParserScope, Platform, ProjectionError, SemanticItem,+ SemanticKind, SemanticParser, SemanticTree,+};+use sha2::{Digest, Sha256};+use std::collections::HashSet;++pub(super) const IDS: &[&str] = &["github", "gitlab", "hackernews", "wikipedia"];+const BODY_BUDGET: usize = 32 * 1024;+const MAX_POSTS: usize = 64;++/// Public reading surfaces only. Each app requires its own observed structural+/// markers; a login page, editor or missing DOM attributes must abstain.+pub(super) struct WebContentParser {+ app: &'static str,+ manifest: ParserManifest,+}++impl WebContentParser {+ pub(super) fn new(app: &'static str) -> Self {+ let profile = builtin_app_profiles().iter().find(|p| p.id == app).unwrap();+ Self {+ app,+ manifest: ParserManifest {+ id: format!("app.{app}.web_content"),+ parser_version: "1".into(),+ schema_version: 1,+ scope: ParserScope::App,+ platforms: vec![Platform::Macos, Platform::Windows, Platform::Linux],+ app_ids: vec![],+ executables: vec![],+ url_patterns: profile.url_patterns.iter().map(|s| (*s).into()).collect(),+ required_attributes: vec![+ AccessibilityAttribute::Title,+ AccessibilityAttribute::Description,+ AccessibilityAttribute::Value,+ AccessibilityAttribute::Children,+ AccessibilityAttribute::DomIdentifier,+ AccessibilityAttribute::DomClasses,+ ],+ app_version: AppVersionRequirement::Any,+ supported_kinds: vec![+ SemanticKind::Conversation,+ SemanticKind::Message,+ SemanticKind::Document,+ ],+ priority: 120,+ },+ }+ }++ fn is_post(&self, tree: &SemanticTree, node: NodeId) -> bool {+ match self.app {+ "github" => [+ "timeline-comment",+ "react-issue-body",+ "react-issue-comment",+ ]+ .iter()+ .any(|c| class(tree, node, c)),+ "gitlab" => {+ class(tree, node, "note-comment") || class(tree, node, "work-item-description")+ }+ // WebKit omits the layout table rows but exposes each comment body+ // and its preceding header as sibling AX groups.+ "hackernews" => {+ class(tree, node, "comtr")+ || class(tree, node, "fatitem")+ || class(tree, node, "commtext")+ || (class(tree, node, "default")+ && tree.descendants(node).any(|n| class(tree, n, "hnuser"))+ && tree.descendants(node).any(|n| {+ class(tree, n, "togg")+ && tree+ .dom_identifier(n)+ .is_some_and(|id| id.bytes().all(|byte| byte.is_ascii_digit()))+ }))+ }+ _ => false,+ }+ }++ fn is_body(&self, tree: &SemanticTree, node: NodeId) -> bool {+ match self.app {+ "github" => class(tree, node, "js-comment-body") || class(tree, node, "markdown-body"),+ "gitlab" => {+ class(tree, node, "note-text") || class(tree, node, "work-item-description")+ }+ "hackernews" => class(tree, node, "commtext") || class(tree, node, "toptext"),+ _ => false,+ }+ }++ fn is_author(&self, tree: &SemanticTree, node: NodeId) -> bool {+ match self.app {+ "github" => {+ class(tree, node, "author")+ || tree.classes(node).any(|c| {+ c.starts_with("IssueBodyHeaderAuthor-module__authorLoginLink__")+ || c.starts_with("ActivityHeader-module__AuthorName__")+ })+ }+ "gitlab" => class(tree, node, "author-name-link"),+ "hackernews" => class(tree, node, "hnuser"),+ _ => false,+ }+ }+}++impl SemanticParser for WebContentParser {+ fn manifest(&self) -> &ParserManifest {+ &self.manifest+ }++ fn parse(+ &self,+ context: &ParseContext<'_>,+ tree: &SemanticTree,+ ) -> Result<ParseOutcome, ProjectionError> {+ let title_node = nodes(tree)+ .filter(|&n| match self.app {+ "github" => {+ (class(tree, n, "markdown-title")+ && (matches!(tree.role(n), Some("AXHeading" | "Heading"))+ || ancestor(tree, n, |p| {+ matches!(tree.role(p), Some("AXHeading" | "Heading"))+ })))+ || tree+ .classes(n)+ .any(|c| c.starts_with("PullRequestHeader-module__inlineTitle__"))+ }+ "gitlab" => class(tree, n, "gl-heading-1"),+ "hackernews" => class(tree, n, "titleline") || class(tree, n, "title"),+ "wikipedia" => tree.dom_identifier(n) == Some("firstHeading"),+ _ => false,+ })+ .find_map(|node| {+ let title_source = if self.app == "hackernews" && class(tree, node, "title") {+ tree.descendants(node)+ .find(|&n| matches!(tree.role(n), Some("AXLink" | "Hyperlink" | "link")))?+ } else {+ node+ };+ let text = content(tree, title_source, 512);+ (!text.is_empty()).then_some((node, text))+ });+ let url = context.app.browser_url.as_deref().unwrap_or_default();+ // HN's item id is its query parameter. Other apps identify documents by+ // path. Never persist tracking parameters, fragments or arbitrary queries.+ let canonical = if self.app == "hackernews" {+ url.split(['&', '#']).next().unwrap_or(url)+ } else {+ url.split(['?', '#']).next().unwrap_or(url)+ };+ if self.app == "wikipedia" && canonical.ends_with("/Main_Page") {+ return Ok(ParseOutcome::NotHandled);+ }+ let scope = format!("{}:{:x}", self.app, Sha256::digest(canonical.as_bytes()));+ let mut root = SemanticItem::new(+ "page",+ if self.app == "wikipedia" {+ SemanticKind::Document+ } else {+ SemanticKind::Conversation+ },+ &scope,+ IdentityQuality::Stable,+ );+ if canonical.len() <= 2048 {+ root.metadata.insert("url".into(), canonical.into());+ }+ // Scrolling can remove the heading from the retained capture. Keep+ // observed posts without guessing a title from comment text.+ if let Some((node, title)) = title_node {+ root.title = Some(title);+ root.source_nodes.push(node);+ }++ if self.app == "wikipedia" {+ let body_node = nodes(tree)+ .find(|&n| {+ class(tree, n, "mw-parser-output")+ && !ancestor(tree, n, |p| excluded(tree, p))+ && ancestor(tree, n, |p| {+ tree.dom_identifier(p) == Some("mw-content-text")+ })+ })+ .or_else(|| {+ // Safari flattens the inert mw-content-text/mw-parser-output+ // wrappers into the article's labelled bodyContent group.+ nodes(tree).find(|&n| {+ tree.dom_identifier(n) == Some("bodyContent")+ && class(tree, n, "vector-body")+ && class(tree, n, "ve-init-mw-desktopArticleTarget-targetContainer")+ && ancestor(tree, n, |p| class(tree, p, "mw-body"))+ && !ancestor(tree, n, |p| excluded(tree, p))+ })+ });+ let Some(body_node) = body_node else {+ return Ok(ParseOutcome::NotHandled);+ };+ let (body, truncated) = extract_content(tree, body_node, BODY_BUDGET);+ if body.is_empty() {+ return Ok(ParseOutcome::NotHandled);+ }+ root.body = Some(body);+ if truncated {+ root.metadata.insert("truncated".into(), "true".into());+ }+ root.source_nodes.push(body_node);+ return Ok(ParseOutcome::Handled(vec![root]));+ }++ let mut items = vec![root];+ let mut remaining = BODY_BUDGET;+ let mut seen = HashSet::new();+ if self.app == "github" {+ append_flat_github_posts(tree, &scope, &mut items, &mut remaining);+ }+ if self.app == "gitlab" {+ append_flat_gitlab_description(tree, &scope, &mut items, &mut remaining);+ }+ for post in nodes(tree).filter(|&n| self.is_post(tree, n)) {+ if items.len() > MAX_POSTS || remaining == 0 {+ items[0].metadata.insert("truncated".into(), "true".into());+ break;+ }+ // Ignore nested post wrappers. Body/author ownership is established+ // by the nearest post container, never by unrelated sidebar links.+ if excluded(tree, post)+ || ancestor(tree, post, |p| self.is_post(tree, p) || excluded(tree, p))+ {+ continue;+ }+ let body_node = tree+ .descendants(post)+ .find(|&n| {+ self.is_body(tree, n)+ && !excluded(tree, n)+ && !ancestor_until(tree, n, post, |p| excluded(tree, p))+ })+ .or_else(|| {+ (self.app == "hackernews" && class(tree, post, "default"))+ .then(|| hn_windows_body(tree, post))+ .flatten()+ })+ .or_else(|| {+ (self.app == "github")+ .then(|| github_windows_body(tree, post))+ .flatten()+ });+ let Some(body_node) = body_node else {+ continue;+ };+ let (body, truncated) = if self.app == "hackernews" && class(tree, post, "default") {+ extract_hn_windows_content(tree, post, remaining)+ } else if self.app == "github" && github_windows_body(tree, post) == Some(body_node) {+ extract_aggregate_content(tree, body_node, remaining)+ } else {+ extract_content(tree, body_node, remaining)+ };+ if body.is_empty() {+ continue;+ }+ let flat_header = (self.app == "hackernews" && class(tree, post, "commtext"))+ .then(|| hn_sibling_header(tree, post))+ .flatten();+ let author = if self.app == "hackernews" && class(tree, post, "default") {+ hn_windows_author(tree, post)+ } else {+ tree.descendants(flat_header.unwrap_or(post))+ .find(|&n| {+ self.is_author(tree, n)+ && n != body_node+ && !excluded(tree, n)+ && !ancestor_until(tree, n, flat_header.unwrap_or(post), |p| {+ p == body_node || excluded(tree, p)+ })+ })+ .or_else(|| {+ (self.app == "gitlab" && class(tree, post, "work-item-description"))+ .then(|| gitlab_work_item_author(tree, post))+ .flatten()+ })+ };+ let native_id = tree+ .dom_identifier(post)+ .filter(|id| !id.is_empty())+ .or_else(|| {+ (self.app == "hackernews" && class(tree, post, "default"))+ .then(|| {+ tree.descendants(post)+ .find(|&n| class(tree, n, "togg"))+ .and_then(|n| tree.dom_identifier(n))+ })+ .flatten()+ })+ .or_else(|| {+ flat_header.and_then(|header| {+ tree.descendants(header)+ .find(|&n| class(tree, n, "togg"))+ .and_then(|n| tree.dom_identifier(n))+ })+ })+ .filter(|id| !id.is_empty() && id.len() <= 128);+ let (key, quality) = match native_id {+ Some(id) => (format!("{scope}:{id}"), IdentityQuality::Stable),+ None => (+ format!("{scope}:node:{}", post.0),+ IdentityQuality::Ephemeral,+ ),+ };+ if !seen.insert(key.clone()) {+ continue;+ }+ remaining -= body.len();+ let mut item = SemanticItem::new(+ format!("post:{}", post.0),+ SemanticKind::Message,+ key,+ quality,+ );+ item.parent_local_id = Some("page".into());+ item.body = Some(body);+ if truncated {+ item.metadata.insert("truncated".into(), "true".into());+ }+ item.source_nodes.push(body_node);+ if let Some(author) = author {+ let value = content(tree, author, 128);+ if !value.is_empty() {+ item.actor = Some(value);+ item.source_nodes.push(author);+ }+ }+ items.push(item);+ }+ if items.len() == 1 {+ return Ok(ParseOutcome::NotHandled);+ }+ Ok(ParseOutcome::Handled(items))+ }+}++/// Chromium UIA flattens each HN comment row into a `default` group. Accept+/// only text after the verified numeric collapse control; separator glyphs and+/// the preceding byline cannot become authored body content.+fn hn_windows_body(tree: &SemanticTree, post: NodeId) -> Option<NodeId> {+ let toggle = hn_windows_toggle(tree, post)?;+ hn_windows_author(tree, post)?;+ tree.descendants(post).find(|&n| {+ n.0 > toggle.0+ && matches!(tree.role(n), Some("Text"))+ && tree+ .text(n)+ .is_some_and(|text| !text.trim().is_empty() && text.trim() != "|")+ && !excluded(tree, n)+ && !ancestor_until(tree, n, post, |p| p != post && class(tree, p, "default"))+ })+}++fn hn_windows_toggle(tree: &SemanticTree, post: NodeId) -> Option<NodeId> {+ tree.descendants(post).find(|&n| {+ class(tree, n, "togg")+ && tree+ .dom_identifier(n)+ .is_some_and(|id| !id.is_empty() && id.bytes().all(|byte| byte.is_ascii_digit()))+ && !ancestor_until(tree, n, post, |p| p != post && class(tree, p, "default"))+ })+}++fn hn_windows_author(tree: &SemanticTree, post: NodeId) -> Option<NodeId> {+ let toggle = hn_windows_toggle(tree, post)?;+ tree.descendants(post).find(|&n| {+ n.0 < toggle.0+ && class(tree, n, "hnuser")+ && tree.text(n).is_some_and(|text| !text.trim().is_empty())+ && !ancestor_until(tree, n, post, |p| p != post && class(tree, p, "default"))+ })+}++fn extract_hn_windows_content(tree: &SemanticTree, post: NodeId, limit: usize) -> (String, bool) {+ let Some(toggle) = hn_windows_toggle(tree, post) else {+ return (String::new(), false);+ };+ if hn_windows_author(tree, post).is_none() {+ return (String::new(), false);+ }+ let mut result = String::new();+ let mut truncated = false;+ for node in tree.descendants(post).filter(|node| node.0 > toggle.0) {+ if excluded(tree, node)+ || ancestor_until(tree, node, post, |p| {+ excluded(tree, p) || (p != post && class(tree, p, "default"))+ })+ || tree.children(node).next().is_some()+ {+ continue;+ }+ let Some(value) = tree.text(node).filter(|value| !value.trim().is_empty()) else {+ continue;+ };+ if !append_bounded(&mut result, value.trim_matches('\n'), limit, &mut truncated) {+ break;+ }+ }+ (result, truncated)+}++/// Chromium RawView retains GitHub's post and visible-content boundary but+/// exposes the inner Markdown container as an unclassified Group. Require the+/// group to be the direct child of `edit-comment-hide` inside a verified post;+/// this excludes timeline activity, sidebar content, and edit controls.+fn github_windows_body(tree: &SemanticTree, post: NodeId) -> Option<NodeId> {+ let visible = tree+ .descendants(post)+ .find(|&n| class(tree, n, "edit-comment-hide") && !excluded(tree, n))?;+ tree.descendants(visible).find(|&n| {+ tree.parent(n) == Some(visible)+ && matches!(tree.role(n), Some("Group"))+ && tree.classes(n).next().is_none()+ && !excluded(tree, n)+ })+}++/// GitLab Chromium UIA exposes the creator in the work-item header, before the+/// description wrapper. The explicit avatar/user classes plus the sibling+/// "by" label establish attribution; a later comment author cannot match.+fn gitlab_work_item_author(tree: &SemanticTree, post: NodeId) -> Option<NodeId> {+ let content = ancestor_node(tree, post, |n| {+ tree.dom_identifier(n) == Some("content-body")+ })?;+ (0..post.0).rev().map(NodeId).find(|&candidate| {+ class(tree, candidate, "gl-avatar-link")+ && class(tree, candidate, "js-user-link")+ && ancestor_node(tree, candidate, |n| n == content).is_some()+ && tree.parent(candidate).is_some_and(|parent| {+ let siblings: Vec<_> = tree.children(parent).collect();+ siblings+ .iter()+ .position(|&node| node == candidate)+ .and_then(|index| index.checked_sub(1))+ .and_then(|index| siblings.get(index))+ .and_then(|&node| tree.text(node))+ .is_some_and(|text| text.trim().eq_ignore_ascii_case("by"))+ })+ })+}++fn ancestor_node(+ tree: &SemanticTree,+ node: NodeId,+ predicate: impl Fn(NodeId) -> bool,+) -> Option<NodeId> {+ let mut parent = tree.parent(node);+ while let Some(candidate) = parent {+ if predicate(candidate) {+ return Some(candidate);+ }+ parent = tree.parent(candidate);+ }+ None+}++fn nodes(tree: &SemanticTree) -> impl Iterator<Item = NodeId> + '_ {+ (0..tree.len()).map(|n| NodeId(n as u32))+}+fn class(tree: &SemanticTree, node: NodeId, expected: &str) -> bool {+ tree.classes(node).any(|c| c == expected)+}++/// Only the immediately preceding sibling header owns a flattened HN body.+/// A recognized collapse control and author are both required, so a nearby+/// story byline or a previous comment cannot be borrowed as attribution.+fn hn_sibling_header(tree: &SemanticTree, body: NodeId) -> Option<NodeId> {+ let parent = tree.parent(body);+ let header = (0..body.0)+ .rev()+ .map(NodeId)+ .find(|&n| tree.parent(n) == parent)?;+ let has_toggle = tree.descendants(header).any(|n| class(tree, n, "togg"));+ let has_author = tree.descendants(header).any(|n| class(tree, n, "hnuser"));+ (has_toggle+ && has_author+ && !excluded(tree, header)+ && !ancestor(tree, header, |p| excluded(tree, p)))+ .then_some(header)+}++/// Safari can omit GitHub's inert comment/body divs. In that shape, accept+/// only a heading with an author and permalink, closed by the matching edit+/// form. The pair bounds authored content without swallowing the timeline.+fn append_flat_github_posts(+ tree: &SemanticTree,+ scope: &str,+ items: &mut Vec<SemanticItem>,+ remaining: &mut usize,+) {+ for header in nodes(tree).filter(|&n| matches!(tree.role(n), Some("AXHeading" | "Heading"))) {+ if ancestor(tree, header, |n| {+ class(tree, n, "timeline-comment")+ || class(tree, n, "react-issue-body")+ || class(tree, n, "react-issue-comment")+ || excluded(tree, n)+ }) {+ continue;+ }+ let Some(permalink) = tree.descendants(header).find(|&n| {+ class(tree, n, "js-timestamp")+ && tree.dom_identifier(n).is_some_and(|id| {+ (id.starts_with("issue-") || id.starts_with("issuecomment-"))+ && id.ends_with("-permalink")+ })+ }) else {+ continue;+ };+ let Some(author) = tree.descendants(header).find(|&n| class(tree, n, "author")) else {+ continue;+ };+ let id = tree+ .dom_identifier(permalink)+ .unwrap()+ .trim_end_matches("-permalink");+ if id.len() > 128 {+ continue;+ }+ let end_id = format!("{id}-edit-form");+ let siblings: Vec<_> = ((header.0 + 1)..tree.len() as u32)+ .map(NodeId)+ .filter(|&n| tree.parent(n) == tree.parent(header))+ .collect();+ let Some(end) = siblings.iter().position(|&n| {+ tree.dom_identifier(n) == Some(end_id.as_str()) && class(tree, n, "js-comment-update")+ }) else {+ continue;+ };+ if siblings[..end]+ .iter()+ .any(|&n| tree.descendants(n).any(|p| class(tree, p, "js-timestamp")))+ {+ continue;+ }+ if items.len() > MAX_POSTS || *remaining == 0 {+ items[0].metadata.insert("truncated".into(), "true".into());+ break;+ }+ let mut body = String::new();+ let mut sources = Vec::new();+ let mut truncated = false;+ for &node in &siblings[..end] {+ if excluded(tree, node)+ || class(tree, node, "details-overlay")+ || class(tree, node, "tooltipped")+ || control(tree, node)+ || matches!(tree.role(node), Some("AXPopUpButton" | "ComboBox"))+ {+ continue;+ }+ let separator = usize::from(!body.is_empty());+ let available = remaining.saturating_sub(body.len() + separator);+ let (text, cut) = extract_content(tree, node, available);+ if !text.is_empty() {+ if separator > 0 {+ body.push('\n');+ }+ body.push_str(&text);+ sources.push(node);+ }+ if cut {+ truncated = true;+ break;+ }+ }+ if body.is_empty() {+ continue;+ }+ *remaining -= body.len();+ let mut item = SemanticItem::new(+ format!("post:{}", header.0),+ SemanticKind::Message,+ format!("{scope}:{id}"),+ IdentityQuality::Stable,+ );+ item.parent_local_id = Some("page".into());+ item.body = Some(body);+ let actor = content(tree, author, 128);+ if !actor.is_empty() {+ item.actor = Some(actor);+ }+ item.source_nodes = sources;+ item.source_nodes.push(author);+ if truncated {+ item.metadata.insert("truncated".into(), "true".into());+ }+ items.push(item);+ }+}++/// WebKit exposes a work item's description as siblings between the title+/// group and attribute sidebar. Require both landmarks in the same content+/// panel; never extend the description into activity or editor content.+fn append_flat_gitlab_description(+ tree: &SemanticTree,+ scope: &str,+ items: &mut Vec<SemanticItem>,+ remaining: &mut usize,+) {+ if nodes(tree).any(|n| class(tree, n, "work-item-description")) {+ return;+ }+ let Some(panel) = nodes(tree).find(|&n| tree.dom_identifier(n) == Some("content-body")) else {+ return;+ };+ let siblings: Vec<_> = tree.children(panel).collect();+ let Some(start) = siblings+ .iter()+ .position(|&n| tree.descendants(n).any(|p| class(tree, p, "gl-heading-1")))+ else {+ return;+ };+ let Some(end) = siblings+ .iter()+ .position(|&n| class(tree, n, "gl-detail-layout-sidebar"))+ else {+ return;+ };+ if start >= end {+ return;+ }+ let mut body = String::new();+ let mut sources = Vec::new();+ let mut truncated = false;+ for &node in &siblings[start + 1..end] {+ if control(tree, node) {+ break;+ }+ if excluded(tree, node) || ancestor(tree, node, |p| excluded(tree, p)) {+ continue;+ }+ let separator = usize::from(!body.is_empty());+ let (text, cut) =+ extract_content(tree, node, remaining.saturating_sub(body.len() + separator));+ if !text.is_empty() {+ if separator > 0 {+ body.push('\n');+ }+ body.push_str(&text);+ sources.push(node);+ }+ if cut {+ truncated = true;+ break;+ }+ }+ if body.is_empty() {+ return;+ }+ *remaining -= body.len();+ let mut item = SemanticItem::new(+ "description",+ SemanticKind::Message,+ format!("{scope}:description"),+ IdentityQuality::Stable,+ );+ item.parent_local_id = Some("page".into());+ item.body = Some(body);+ item.source_nodes = sources;+ if let Some(author) = tree+ .descendants(siblings[start])+ .find(|&n| class(tree, n, "gl-avatar-link") && class(tree, n, "js-user-link"))+ {+ let actor = content(tree, author, 128);+ if !actor.is_empty() {+ item.actor = Some(actor);+ }+ item.source_nodes.push(author);+ }+ if truncated {+ item.metadata.insert("truncated".into(), "true".into());+ }+ items.push(item);+}+fn ancestor(tree: &SemanticTree, node: NodeId, predicate: impl Fn(NodeId) -> bool) -> bool {+ let mut parent = tree.parent(node);+ while let Some(p) = parent {+ if predicate(p) {+ return true;+ }+ parent = tree.parent(p);+ }+ false+}+fn control(tree: &SemanticTree, node: NodeId) -> bool {+ matches!(+ tree.role(node),+ Some("AXButton" | "Button" | "button" | "AXMenuItem" | "MenuItem")+ )+}+fn excluded(tree: &SemanticTree, node: NodeId) -> bool {+ duplicate_table_axis(tree, node)+ || (matches!(tree.role(node), Some("AXLink" | "Hyperlink" | "link"))+ && tree+ .text(node)+ .is_some_and(|text| text.eq_ignore_ascii_case("edit"))+ && tree+ .description(node)+ .is_some_and(|description| description.starts_with("Edit section:")))+ || tree.dom_identifier(node) == Some("siteSub")+ || matches!(+ tree.role(node),+ Some("AXTextArea" | "Edit" | "TextBox" | "AXTextField" | "textbox")+ )+ || [+ "js-preview-body",+ "timeline-comment-actions",+ "mw-editsection",+ "sitebit",+ "reply",+ "description-more",+ "anchor",+ "navbox",+ "sidebar",+ "portalbox",+ ]+ .iter()+ .any(|c| class(tree, node, c))+}++/// WebKit exposes the same cells through rows, columns and a header group.+/// Preserve authored repetition within rows, but do not read alternate axes.+fn duplicate_table_axis(tree: &SemanticTree, node: NodeId) -> bool {+ let Some(table) = tree+ .parent(node)+ .filter(|&p| tree.role(p) == Some("AXTable"))+ else {+ return false;+ };+ if !tree.children(table).any(|n| tree.role(n) == Some("AXRow")) {+ return false;+ }+ tree.role(node) == Some("AXColumn")+ || (tree.role(node) == Some("AXGroup")+ && tree.children(node).next().is_some()+ && tree.children(node).all(|n| tree.role(n) == Some("AXCell")))+}++/// Read only leaves so an aggregate AX label and its descendants, or nested+/// markdown-body wrappers, cannot repeat the same paragraph. Preserve code+/// indentation and internal newlines; identical authored paragraphs stay intact.+fn content(tree: &SemanticTree, root: NodeId, limit: usize) -> String {+ extract_content(tree, root, limit).0+}++fn extract_content(tree: &SemanticTree, root: NodeId, limit: usize) -> (String, bool) {+ let mut result = String::new();+ let mut truncated = false;+ for node in tree.descendants(root) {+ if excluded(tree, node) || ancestor_until(tree, node, root, |p| excluded(tree, p)) {+ continue;+ }+ if tree.children(node).next().is_some() {+ continue;+ }+ let value = tree+ .text(node)+ .or_else(|| tree.title(node))+ .or_else(|| {+ tree.description(node).filter(|description| {+ !matches!(+ *description,+ "text"+ | "link"+ | "group"+ | "heading"+ | "image"+ | "button"+ | "cell"+ | "row"+ | "column"+ )+ })+ })+ .or_else(|| tree.value(node));+ let Some(value) = value.filter(|v| !v.trim().is_empty()) else {+ continue;+ };+ // Suppress only recognized labels under controls. Reconstructed AX+ // trees can reparent real authored text under a button, so never drop+ // its whole subtree merely because the ancestor is a control.+ if matches!(+ value.trim(),+ "Copy" | "Copy code" | "Edit" | "Reply" | "Read more"+ ) && (control(tree, node) || ancestor_until(tree, node, root, |p| control(tree, p)))+ {+ continue;+ }+ let value = value.trim_matches('\n');+ let separator = usize::from(!result.is_empty());+ if result.len() + separator >= limit {+ truncated = true;+ break;+ }+ if separator != 0 {+ result.push('\n');+ }+ let mut end = value.len().min(limit - result.len());+ while !value.is_char_boundary(end) {+ end -= 1;+ }+ result.push_str(&value[..end]);+ if end < value.len() {+ truncated = true;+ break;+ }+ }+ (result, truncated)+}++/// Chromium UIA can expose one aggregate Text node with link fragments as its+/// children. Read the highest authored text at each branch so the aggregate is+/// preserved without repeating its descendants.+fn extract_aggregate_content(tree: &SemanticTree, root: NodeId, limit: usize) -> (String, bool) {+ let mut result = String::new();+ let mut truncated = false;+ let mut skip_descendants_of: Option<NodeId> = None;+ for node in tree.descendants(root) {+ if skip_descendants_of+ .is_some_and(|aggregate| ancestor_node(tree, node, |n| n == aggregate).is_some())+ {+ continue;+ }+ skip_descendants_of = None;+ if excluded(tree, node) || ancestor_until(tree, node, root, |p| excluded(tree, p)) {+ continue;+ }+ let Some(value) = tree.text(node).filter(|value| !value.trim().is_empty()) else {+ continue;+ };+ let value = value.trim_matches('\n');+ if matches!(+ value.trim(),+ "Copy" | "Copy code" | "Edit" | "Reply" | "Read more"+ ) && (control(tree, node) || ancestor_until(tree, node, root, |p| control(tree, p)))+ {+ continue;+ }+ let mut saw_descendant_text = false;+ let contains_all_descendant_text = tree+ .descendants(node)+ .skip(1)+ .filter(|&descendant| {+ !excluded(tree, descendant)+ && !ancestor_until(tree, descendant, node, |p| excluded(tree, p))+ })+ .filter_map(|descendant| tree.text(descendant))+ .map(str::trim)+ .filter(|text| !text.is_empty())+ .all(|descendant| {+ saw_descendant_text = true;+ value.contains(descendant)+ });+ if saw_descendant_text && contains_all_descendant_text {+ skip_descendants_of = Some(node);+ }+ if !append_bounded(&mut result, value, limit, &mut truncated) {+ break;+ }+ }+ (result, truncated)+}++fn append_bounded(result: &mut String, value: &str, limit: usize, truncated: &mut bool) -> bool {+ let separator = usize::from(!result.is_empty());+ if result.len() + separator >= limit {+ *truncated = true;+ return false;+ }+ if separator != 0 {+ result.push('\n');+ }+ let mut end = value.len().min(limit - result.len());+ while !value.is_char_boundary(end) {+ end -= 1;+ }+ result.push_str(&value[..end]);+ if end < value.len() {+ *truncated = true;+ return false;+ }+ true+}+fn ancestor_until(+ tree: &SemanticTree,+ node: NodeId,+ root: NodeId,+ predicate: impl Fn(NodeId) -> bool,+) -> bool {+ if node == root {+ return false;+ }+ let mut parent = tree.parent(node);+ while let Some(p) = parent {+ if predicate(p) {+ return true;+ }+ if p == root {+ break;+ }+ parent = tree.parent(p);+ }+ false+}diff --git a/crates/screenpipe-semantic/tests/builtin_catalog.rs b/crates/screenpipe-semantic/tests/builtin_catalog.rsindex 4b5fe52f0..db4e58407 100644--- a/crates/screenpipe-semantic/tests/builtin_catalog.rs+++ b/crates/screenpipe-semantic/tests/builtin_catalog.rs@@ -30,7 +30,10 @@ const EXPECTED_TARGETS: &[&str] = &["gemini_desktop","geminiweb","ghostty",+ "github",+ "gitlab","gmail",+ "hackernews","hey","iterm2","mail",@@ -61,6 +64,7 @@ const EXPECTED_TARGETS: &[&str] = &["warp","whatsapp","whatsapp_web",+ "wikipedia","windowsconsole","windowsterminal","windsurf",@@ -88,21 +92,21 @@ fn identity_for(profile: &screenpipe_semantic::parsers::BuiltinAppProfile) -> Ap}#[test]-fn catalog_exactly_matches_all_56_targets() {+fn catalog_exactly_matches_all_60_targets() {let actual: BTreeSet<_> = builtin_app_profiles().iter().map(|profile| profile.id).collect();let expected: BTreeSet<_> = EXPECTED_TARGETS.iter().copied().collect();- assert_eq!(builtin_app_profiles().len(), 56);- assert_eq!(actual.len(), 56, "profile ids must be unique");+ assert_eq!(builtin_app_profiles().len(), 60);+ assert_eq!(actual.len(), 60, "profile ids must be unique");assert_eq!(actual, expected);}#[test]fn every_profile_has_identity_family_and_bounded_registry_coverage() {let registry = builtin_parser_registry().expect("built-in manifests must compile");- assert_eq!(registry.len(), 23);+ assert_eq!(registry.len(), 27);for profile in builtin_app_profiles() {assert!(!profile.families.is_empty(), "{} has no family", profile.id);let unique_families: HashSet<_> = profile.families.iter().copied().collect();@@ -147,7 +151,11 @@ fn every_profile_has_identity_family_and_bounded_registry_coverage() {));assert_eq!(plan.parser_ids.len(),- profile.families.len() + override_count,+ if matches!(profile.id, "github" | "gitlab" | "hackernews" | "wikipedia") {+ 1+ } else {+ profile.families.len() + override_count+ },"{} matched unexpected parser candidates",profile.id);diff --git a/crates/screenpipe-semantic/tests/capture_adapter.rs b/crates/screenpipe-semantic/tests/capture_adapter.rsindex 2d569f594..22a90fb84 100644--- a/crates/screenpipe-semantic/tests/capture_adapter.rs+++ b/crates/screenpipe-semantic/tests/capture_adapter.rs@@ -251,3 +251,18 @@ fn serialized_capture_shape_omits_absent_optional_fields() {assert_eq!(json, r#"{"role":"AXWindow","text":"title","depth":0}"#);assert!(!json.contains("null"));}++#[test]+fn explicit_native_offscreen_geometry_survives_discarded_normalized_bounds() {+ let mut nodes = vec![node("AXStaticText", "off-window text", 0)];+ nodes[0].on_screen = Some(false);+ nodes[0].offscreen_geometry = true;+ let adapted = adapt_captured_accessibility_tree(&nodes, TreeBudget::default()).unwrap();+ let root = adapted.tree.roots().next().unwrap();+ assert_eq!(adapted.tree.text(root), None);+ assert_eq!(adapted.stats.known_offscreen_nodes, 1);+ // Contradictory evidence must not discard a node marked on-screen.+ nodes[0].on_screen = Some(true);+ let adapted = adapt_captured_accessibility_tree(&nodes, TreeBudget::default()).unwrap();+ assert_eq!(adapted.tree.text(root), Some("off-window text"));+}diff --git a/crates/screenpipe-semantic/tests/context_efficiency.rs b/crates/screenpipe-semantic/tests/context_efficiency.rsindex f3b10bcd3..4c7eb67b0 100644--- a/crates/screenpipe-semantic/tests/context_efficiency.rs+++ b/crates/screenpipe-semantic/tests/context_efficiency.rs@@ -9,8 +9,8 @@ mod context_eval;fn semantic_context_is_smaller_than_raw_without_losing_task_facts() {let report = context_eval::evaluate_suite().expect("context eval must run");- assert_eq!(report.catalog_profiles, 56);- assert_eq!(report.parser_implementations, 23);+ assert_eq!(report.catalog_profiles, 60);+ assert_eq!(report.parser_implementations, 27);assert_eq!(report.representative_cases, 10);assert_eq!(report.totals.raw_json.offscreen_distractors_retained, 10);assert_eq!(diff --git a/crates/screenpipe-semantic/tests/context_quality.rs b/crates/screenpipe-semantic/tests/context_quality.rsnew file mode 100644index 000000000..0b18bf3c3--- /dev/null+++ b/crates/screenpipe-semantic/tests/context_quality.rs@@ -0,0 +1,22 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++#[path = "../evals/quality/mod.rs"]+mod quality;++#[test]+fn context_preserves_observed_facts_without_inventing_state_or_copying_controls() {+ for report in quality::evaluate().expect("quality fixtures must parse") {+ assert!(+ report.failures.is_empty(),+ "{}: {:?}",+ report.id,+ report.failures+ );+ assert!(+ report.context_tokens < report.raw_tokens,+ "{} must remain compact",+ report.id+ );+ }+}diff --git a/crates/screenpipe-semantic/tests/family_parsers.rs b/crates/screenpipe-semantic/tests/family_parsers.rsindex 07f80d42e..c51fce02a 100644--- a/crates/screenpipe-semantic/tests/family_parsers.rs+++ b/crates/screenpipe-semantic/tests/family_parsers.rs@@ -107,6 +107,34 @@ fn conversation_fixture_preserves_sender_relationships() {);}+#[test]+fn nested_task_wrappers_keep_leaf_tasks_and_independent_siblings() {+ let source = serde_json::json!({+ "app": { "platform": "windows", "executable": "Todoist.exe", "display_name": "Todoist" },+ "nodes": [+ { "parent": null, "role": "Window" },+ { "parent": 0, "role": "Group", "classes": ["task-row"] },+ { "parent": 1, "role": "Text", "value": "Container label" },+ { "parent": 1, "role": "Group" },+ { "parent": 3, "role": "Group", "classes": ["task-row"] },+ { "parent": 4, "role": "Text", "value": "First child" },+ { "parent": 3, "role": "Group", "classes": ["task-row"] },+ { "parent": 6, "role": "Text", "value": "Second child" },+ { "parent": 0, "role": "Group", "classes": ["task-row"] },+ { "parent": 8, "role": "Text", "value": "Independent sibling" }+ ]+ })+ .to_string();+ let items = handled(&source, "family.task");+ assert_eq!(+ items+ .iter()+ .map(|item| item.title.as_deref().unwrap())+ .collect::<Vec<_>>(),+ vec!["First child", "Second child", "Independent sibling"]+ );+}+#[test]fn mail_fixture_extracts_subject_sender_and_body() {let items = handled(diff --git a/crates/screenpipe-semantic/tests/messaging_contract.rs b/crates/screenpipe-semantic/tests/messaging_contract.rsindex 1d9e07e99..b6fd24a99 100644--- a/crates/screenpipe-semantic/tests/messaging_contract.rs+++ b/crates/screenpipe-semantic/tests/messaging_contract.rs@@ -220,5 +220,5 @@ fn messaging_parser_versions_invalidate_stale_parse_runs() {.collect::<Vec<_>>();assert!(versions.contains(&("app.macos.slack.content_list", "2")));- assert!(versions.contains(&("family.conversation", "4")));+ assert!(versions.contains(&("family.conversation", "6")));}diff --git a/crates/screenpipe-semantic/tests/web_app_coverage.rs b/crates/screenpipe-semantic/tests/web_app_coverage.rsnew file mode 100644index 000000000..b94aceb07--- /dev/null+++ b/crates/screenpipe-semantic/tests/web_app_coverage.rs@@ -0,0 +1,260 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++//! Structural replay of existing reviewed fixtures through missing browser routes.+//! These prove routing and semantic parity, not live authenticated-app coverage.+use screenpipe_semantic::{+ parsers::builtin_parser_registry, AppIdentity, NodeId, OutputBudget, ParseContext, Platform,+ SemanticItem, SemanticNodeInput, SemanticTree, SemanticTreeBuilder, TreeBudget,+ ValidatedParseOutcome,+};++fn fixture(source: &str) -> (AppIdentity, SemanticTree) {+ let value: serde_json::Value = serde_json::from_str(source).unwrap();+ let mut builder = SemanticTreeBuilder::new(TreeBudget::default());+ for node in value["nodes"].as_array().unwrap() {+ let field = |key: &str| node[key].as_str();+ let classes: Vec<_> = node["classes"]+ .as_array()+ .into_iter()+ .flatten()+ .filter_map(|v| v.as_str())+ .collect();+ builder+ .push(+ node["parent"].as_u64().map(|id| NodeId(id as u32)),+ SemanticNodeInput {+ role: field("role").unwrap(),+ text: field("text"),+ title: field("title"),+ value: field("value"),+ description: field("description"),+ identifier: field("identifier"),+ dom_identifier: field("dom_identifier"),+ classes: &classes,+ ..Default::default()+ },+ )+ .unwrap();+ }+ (+ serde_json::from_value(value["app"].clone()).unwrap(),+ builder.finish(),+ )+}++fn browser(url: &str, platform: Platform) -> AppIdentity {+ AppIdentity {+ platform,+ app_id: None,+ executable: Some("chrome.exe".into()),+ display_name: "Chrome".into(),+ version: None,+ browser_url: Some(url.into()),+ }+}++#[test]+fn web_routes_preserve_existing_structural_extraction_on_all_platforms() {+ let registry = builtin_parser_registry().unwrap();+ for (source, urls, expected_app) in [+ (+ include_str!("fixtures/families/slack_conversation.json"),+ vec!["https://app.slack.com/client/TEXAMPLE/CEXAMPLE"],+ "Slack",+ ),+ (+ include_str!("fixtures/families/teams_windows_chat.json"),+ vec!["https://teams.microsoft.com/v2/"],+ "Microsoft Teams",+ ),+ (+ include_str!("fixtures/families/notion_document.json"),+ vec![+ "https://www.notion.so/workspace/page",+ "https://notion.so/page",+ "https://app.notion.com/p/acme",+ "https://example.notion.site/page",+ ],+ "Notion",+ ),+ (+ include_str!("fixtures/families/todoist_tasks.json"),+ vec!["https://to-do.office.com/tasks/"],+ "Microsoft To Do",+ ),+ (+ include_str!("fixtures/editor/vscode_macos.json"),+ vec![+ "https://vscode.dev/",+ "https://insiders.vscode.dev/",+ "https://github.dev/example/repository",+ ],+ "VS Code",+ ),+ ] {+ let (native, tree) = fixture(source);+ let parse = |app: &AppIdentity| {+ let context = ParseContext {+ frame_id: 1,+ captured_at_unix_ms: 1_700_000_000_000,+ utc_offset_minutes: None,+ locale_hint: None,+ app,+ input_content_hash: 1,+ };+ let result = registry.parse(&context, &tree, OutputBudget::default());+ assert!(result.failures.is_empty(), "{app:?}: {:?}", result.failures);+ match result.outcome {+ ValidatedParseOutcome::Handled(projection) => projection.into_items(),+ outcome => panic!("{app:?}: {outcome:?}"),+ }+ };+ let expected = parse(&native);+ let facts = |items: &[SemanticItem]| {+ items+ .iter()+ .map(|item| {+ (+ item.kind,+ item.title.clone(),+ item.body.clone(),+ item.actor.clone(),+ item.status.clone(),+ )+ })+ .collect::<Vec<_>>()+ };+ for url in urls {+ for platform in [Platform::Macos, Platform::Windows, Platform::Linux] {+ let actual = parse(&browser(url, platform));+ assert_eq!(facts(&actual), facts(&expected), "{url} {platform:?}");+ assert_eq!(+ actual[0].metadata.get("app").map(String::as_str),+ Some(expected_app)+ );+ }+ }+ }+}++#[test]+fn url_markers_in_query_strings_do_not_change_the_selected_app_profile() {+ let (_, tree) = fixture(include_str!("fixtures/families/gmail_thread.json"));+ let app = browser(+ "https://mail.google.com/mail/u/0/#inbox?link=https://app.fastmail.com/mail/",+ Platform::Macos,+ );+ let context = ParseContext {+ frame_id: 1,+ captured_at_unix_ms: 0,+ utc_offset_minutes: None,+ locale_hint: None,+ app: &app,+ input_content_hash: 1,+ };+ let result = builtin_parser_registry()+ .unwrap()+ .parse(&context, &tree, OutputBudget::default());+ let ValidatedParseOutcome::Handled(projection) = result.outcome else {+ panic!("expected mail");+ };+ assert_eq!(+ projection.items()[0]+ .metadata+ .get("app")+ .map(String::as_str),+ Some("Gmail")+ );+}++#[test]+fn lookalike_origins_and_marketing_pages_do_not_route_to_app_parsers() {+ let registry = builtin_parser_registry().unwrap();+ for url in [+ "https://app.slack.com.evil.test/client/T/C",+ "https://evil.test/app.slack.com/client/T/C",+ "https://[email protected]/client/T/C",+ "https://app.slack.com/marketing",+ "https://teams.microsoft.com.evil.test/v2/",+ "https://www.notion.so.evil.test/page",+ "https://example.notion.site.evil.test/page",+ "https://notnotion.so/page",+ "https://evil.test/?next=https://vscode.dev/",+ "https://notgithub.dev/repo",+ "https://to-do.office.com.evil.test/tasks/",+ "https://www.notion.com/pricing",+ "https://evilsharepoint.com/sites/a/_layouts/15/Doc.aspx",+ "https://eviloffice.com/files/a.docx",+ "https://evilofficeapps.live.com/we/wordeditorframe",+ "https://evilonedrive.live.com/files/a/_layouts/15/Doc.aspx",+ ] {+ assert!(+ registry+ .capture_plan(&browser(url, Platform::Windows))+ .is_none(),+ "{url}"+ );+ }+}++#[test]+fn supported_web_identity_does_not_publish_login_chrome() {+ let registry = builtin_parser_registry().unwrap();+ let mut builder = SemanticTreeBuilder::new(TreeBudget::default());+ let root = builder+ .push(+ None,+ SemanticNodeInput {+ role: "Document",+ title: Some("Sign in"),+ ..Default::default()+ },+ )+ .unwrap();+ builder+ .push(+ Some(root),+ SemanticNodeInput {+ role: "Edit",+ title: Some("Email address"),+ ..Default::default()+ },+ )+ .unwrap();+ builder+ .push(+ Some(root),+ SemanticNodeInput {+ role: "Button",+ title: Some("Continue"),+ ..Default::default()+ },+ )+ .unwrap();+ let tree = builder.finish();+ for url in [+ "https://app.slack.com/client/T/C",+ "https://teams.microsoft.com/v2/",+ "https://app.notion.com/login",+ "https://to-do.office.com/tasks/login",+ "https://vscode.dev/",+ ] {+ let app = browser(url, Platform::Windows);+ let context = ParseContext {+ frame_id: 1,+ captured_at_unix_ms: 0,+ utc_offset_minutes: None,+ locale_hint: None,+ app: &app,+ input_content_hash: 1,+ };+ assert_eq!(+ registry+ .parse(&context, &tree, OutputBudget::default())+ .outcome,+ ValidatedParseOutcome::NotHandled,+ "{url}"+ );+ }+}diff --git a/crates/screenpipe-semantic/tests/web_content.rs b/crates/screenpipe-semantic/tests/web_content.rsnew file mode 100644index 000000000..c60357bf9--- /dev/null+++ b/crates/screenpipe-semantic/tests/web_content.rs@@ -0,0 +1,657 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++#[allow(dead_code)]+#[path = "../evals/quality/mod.rs"]+mod quality;++use screenpipe_semantic::{+ adapt_captured_accessibility_tree, parsers::builtin_parser_registry, AppIdentity,+ CapturedAccessibilityNode, OutputBudget, ParseContext, Platform, SemanticItem, TreeBudget,+ ValidatedParseOutcome,+};+use serde::Deserialize;++#[derive(Clone, Deserialize)]+struct Case {+ id: String,+ app: AppIdentity,+ nodes: Vec<CapturedAccessibilityNode>,+}+fn cases() -> Vec<Case> {+ serde_json::from_str(include_str!("../evals/web_content/cases.json")).unwrap()+}+fn parse(case: &Case) -> screenpipe_semantic::ParserChainResult {+ let tree = adapt_captured_accessibility_tree(&case.nodes, TreeBudget::default()).unwrap();+ builtin_parser_registry().unwrap().parse(+ &ParseContext {+ frame_id: 7,+ captured_at_unix_ms: 0,+ utc_offset_minutes: None,+ locale_hint: None,+ app: &case.app,+ input_content_hash: 1,+ },+ &tree.tree,+ OutputBudget::default(),+ )+}+fn items(case: &Case) -> Vec<SemanticItem> {+ let result = parse(case);+ assert!(+ result.failures.is_empty(),+ "{}: {:?}",+ case.id,+ result.failures+ );+ assert_eq!(+ result.selected_parser_id.as_deref(),+ Some(match case.id.as_str() {+ "github_issue" | "github_pull" => "app.github.web_content",+ "gitlab" => "app.gitlab.web_content",+ "hackernews" => "app.hackernews.web_content",+ "wikipedia" => "app.wikipedia.web_content",+ _ => unreachable!(),+ })+ );+ let ValidatedParseOutcome::Handled(p) = result.outcome else {+ panic!("{} not handled", case.id);+ };+ p.into_items()+}++#[test]+fn preserves_fields_attribution_and_context_without_page_chrome_on_all_platforms() {+ for platform in ["macos", "windows", "linux"] {+ let mut gold: serde_json::Value =+ serde_json::from_str(include_str!("../evals/web_content/cases.json")).unwrap();+ for case in gold.as_array_mut().unwrap() {+ case["app"]["platform"] = platform.into();+ for node in case["nodes"].as_array_mut().unwrap() {+ if platform != "macos" {+ node["role"] = match node["role"].as_str().unwrap() {+ "AXGroup" => "Group",+ "AXStaticText" => "Text",+ "AXHeading" => "Heading",+ "AXLink" => "Hyperlink",+ "AXButton" => "Button",+ "AXTextArea" => "Edit",+ other => other,+ }+ .into();+ }+ }+ }+ for report in quality::evaluate_cases(&gold.to_string()).unwrap() {+ assert!(+ report.failures.is_empty(),+ "{platform}/{}: {:?}",+ report.id,+ report.failures+ );+ assert!(report.context_tokens < report.raw_tokens);+ }+ }+ for (case, count) in cases().iter().zip([3, 2, 3, 4, 1]) {+ assert_eq!(+ items(case).len(),+ count,+ "{} emitted extra records",+ case.id+ );+ }+}++#[test]+fn unsupported_origins_and_surfaces_never_route() {+ let registry = builtin_parser_registry().unwrap();+ let mut app = cases()[0].app.clone();+ for url in [+ "https://github.com.evil.test/org/repo/issues/1",+ "https://example.org/?next=https://github.com/org/repo/issues/1",+ "https://github.com/org/repo/issues/new",+ "https://github.com/org/repo/pull/1/files",+ "https://github.com/org/repo/issues/1extra",+ "https://gitlab.com.evil.test/org/repo/-/issues/1",+ "https://gitlab.com/org/repo/-/merge_requests/1",+ "https://news.ycombinator.com/item?id=1extra",+ "https://news.ycombinator.com/newest",+ "https://en.wikipedia.org/wiki/Special:Search",+ "https://en.wikipedia.org/wiki/Talk%3AComputer",+ "https://en.wikipedia.org.evil.test/wiki/Computer",+ ] {+ app.browser_url = Some(url.into());+ assert!(+ registry.capture_plan(&app).is_none(),+ "unexpected route: {url}"+ );+ }+}++#[test]+fn missing_contracts_and_draft_only_pages_abstain_without_family_fallback() {+ for mut case in cases() {+ for node in &mut case.nodes {+ node.class_name = None;+ node.dom_identifier = None;+ }+ assert_eq!(+ parse(&case).outcome,+ ValidatedParseOutcome::NotHandled,+ "{}",+ case.id+ );+ }+ let mut case = cases().remove(0);+ // Even recognized markdown containers underneath an editor are draft text.+ case.nodes = vec![+ node(0, "AXTextArea", "", "UNSENT_DRAFT"),+ node(1, "AXGroup", "react-issue-comment", ""),+ node(2, "AXGroup", "markdown-body", ""),+ node(3, "AXStaticText", "", "UNSENT_DRAFT"),+ ];+ assert_eq!(parse(&case).outcome, ValidatedParseOutcome::NotHandled);+ case.nodes = vec![+ node(0, "AXWebArea", "", ""),+ node(1, "AXGroup", "timeline-comment", ""),+ node(2, "AXGroup", "js-preview-body", ""),+ node(3, "AXGroup", "markdown-body", ""),+ node(4, "AXStaticText", "", "UNSENT_PREVIEW"),+ ];+ assert_eq!(parse(&case).outcome, ValidatedParseOutcome::NotHandled);+}++#[test]+fn scrolled_threads_keep_visible_posts_without_inventing_titles_or_authors() {+ let mut case = cases().remove(0);+ for node in &mut case.nodes {+ if node.class_name.as_deref().is_some_and(|c| {+ c == "markdown-title"+ || c.starts_with("ActivityHeader-")+ || c.starts_with("IssueBodyHeaderAuthor-")+ }) {+ node.text.clear();+ }+ }+ let parsed = items(&case);+ assert_eq!(parsed[0].title, None);+ assert_eq!(parsed[1].actor, None);+ assert_eq!(+ parsed[2].actor, None,+ "quoted author names must not become attribution"+ );+ assert!(parsed[2].body.as_ref().unwrap().contains("Quoted Person"));+}++#[test]+fn native_post_identity_survives_position_and_query_changes_but_not_different_threads() {+ let mut case = cases().remove(3);+ let before = items(&case);+ case.app.browser_url =+ Some("https://news.ycombinator.com/item?id=42&tracking=PRIVATE_TOKEN#comment".into());+ case.nodes+ .insert(2, node(1, "AXStaticText", "", "NEW_SIDEBAR"));+ let after = items(&case);+ assert_eq!(before[0].item_key, after[0].item_key);+ assert_eq!(before[2].item_key, after[2].item_key);+ assert!(!serde_json::to_string(&after)+ .unwrap()+ .contains("PRIVATE_TOKEN"));+ // Equal bodies from different people remain distinct messages.+ assert_eq!(before[2].body, before[3].body);+ assert_ne!(before[2].item_key, before[3].item_key);+ case.app.browser_url = Some("https://news.ycombinator.com/item?id=43".into());+ assert_ne!(before[2].item_key, items(&case)[2].item_key);+}++#[test]+fn repeated_native_post_wrappers_do_not_repeat_messages() {+ let mut case = cases().remove(1);+ let copy = case.nodes[3..10].to_vec();+ case.nodes.extend(copy);+ assert_eq!(items(&case).len(), 2);+}++#[test]+fn large_multibyte_discussions_respect_shared_budget_and_mark_truncation() {+ let mut case = cases().remove(0);+ case.nodes = vec![node(0, "AXWebArea", "", "")];+ for index in 0..100 {+ let mut post = node(1, "AXGroup", "react-issue-comment", "");+ post.dom_identifier = Some(format!("issuecomment-{index}"));+ case.nodes.push(post);+ case.nodes.push(node(+ 2,+ "AXLink",+ "ActivityHeader-module__AuthorName__test",+ &"名".repeat(100),+ ));+ case.nodes.push(node(2, "AXGroup", "markdown-body", ""));+ case.nodes+ .push(node(3, "AXStaticText", "", &"語".repeat(700)));+ }+ for platform in [Platform::Macos, Platform::Windows, Platform::Linux] {+ case.app.platform = platform;+ let parsed = items(&case);+ assert!(parsed.len() <= 65);+ assert!(+ parsed+ .iter()+ .filter_map(|i| i.body.as_ref())+ .map(String::len)+ .sum::<usize>()+ <= 32 * 1024+ );+ assert!(parsed+ .iter()+ .any(|i| i.metadata.get("truncated").is_some_and(|s| s == "true")));+ }+}++#[test]+fn authored_content_reparented_under_controls_is_preserved() {+ let mut case = cases().remove(1);+ let body = case+ .nodes+ .iter()+ .position(|n| n.text == "Review this report.")+ .unwrap();+ case.nodes[body].depth += 1;+ case.nodes.insert(body, node(3, "AXButton", "", "Copy"));+ let parsed = items(&case);+ assert_eq!(+ parsed[1].body.as_deref(),+ Some("Review this report.\nreport.csv")+ );+}++fn node(depth: u8, role: &str, class: &str, text: &str) -> CapturedAccessibilityNode {+ CapturedAccessibilityNode {+ depth,+ role: role.into(),+ text: text.into(),+ class_name: Some(class.into()),+ ..Default::default()+ }+}++#[test]+fn webkit_flattened_comments_keep_local_authors_and_stable_ids() {+ let mut case = cases().remove(3);+ case.nodes = vec![+ node(0, "AXGroup", "title", ""),+ node(1, "AXLink", "", "Synthetic discussion"),+ node(0, "AXGroup", "subtext", ""),+ node(1, "AXLink", "hnuser", "Story submitter"),+ node(0, "AXGroup", "", ""),+ node(1, "AXLink", "hnuser", "Alice"),+ node(1, "AXLink", "togg clicky", "Collapse"),+ node(0, "AXGroup", "commtext c00", ""),+ node(1, "AXStaticText", "", "First authored comment"),+ node(0, "AXGroup", "commtext c00", ""),+ node(1, "AXStaticText", "", "Comment with no retained header"),+ ];+ case.nodes[6].dom_identifier = Some("123".into());+ let parsed = items(&case);+ assert_eq!(parsed.len(), 3);+ assert_eq!(parsed[0].title.as_deref(), Some("Synthetic discussion"));+ assert_eq!(parsed[1].actor.as_deref(), Some("Alice"));+ assert_eq!(+ parsed[1].identity_quality,+ screenpipe_semantic::IdentityQuality::Stable+ );+ assert_eq!(parsed[2].actor, None, "do not borrow a previous author");+ assert_eq!(+ parsed[2].body.as_deref(),+ Some("Comment with no retained header")+ );+}++#[test]+fn webkit_github_comment_requires_matching_end_and_excludes_timeline() {+ let mut case = cases().remove(1);+ case.nodes = vec![+ node(+ 0,+ "AXHeading",+ "PullRequestHeader-module__inlineTitle__example",+ "",+ ),+ node(1, "AXStaticText", "", "Synthetic pull request"),+ node(0, "AXHeading", "", ""),+ node(1, "AXLink", "author", "Alice"),+ node(1, "AXLink", "js-timestamp", "Yesterday"),+ node(0, "AXGroup", "tooltipped", "Collaborator"),+ node(0, "AXGroup", "", ""),+ node(1, "AXStaticText", "", "Authored description"),+ node(0, "AXGroup", "js-comment-update", ""),+ node(+ 0,+ "AXGroup",+ "TimelineItem-body",+ "Merged an unrelated commit",+ ),+ ];+ case.nodes[4].dom_identifier = Some("issue-42-permalink".into());+ case.nodes[8].dom_identifier = Some("issue-42-edit-form".into());+ let parsed = items(&case);+ assert_eq!(parsed.len(), 2);+ assert_eq!(parsed[0].title.as_deref(), Some("Synthetic pull request"));+ assert_eq!(parsed[1].actor.as_deref(), Some("Alice"));+ assert_eq!(parsed[1].body.as_deref(), Some("Authored description"));+ case.nodes[8].dom_identifier = Some("issue-99-edit-form".into());+ assert!(matches!(+ parse(&case).outcome,+ ValidatedParseOutcome::NotHandled+ ));+}++#[test]+fn webkit_wikipedia_body_excludes_sidebar_and_offwindow_content() {+ let mut case = cases().remove(4);+ case.nodes = vec![+ node(0, "AXGroup", "mw-body", ""),+ node(1, "AXHeading", "", "Synthetic article"),+ node(+ 1,+ "AXGroup",+ "vector-body ve-init-mw-desktopArticleTarget-targetContainer",+ "",+ ),+ node(2, "AXGroup", "sidebar", ""),+ node(3, "AXStaticText", "", "Unrelated navigation"),+ node(2, "AXStaticText", "", "Visible article paragraph"),+ node(2, "AXStaticText", "", "Offscreen article paragraph"),+ ];+ case.nodes[1].dom_identifier = Some("firstHeading".into());+ case.nodes[2].dom_identifier = Some("bodyContent".into());+ case.nodes[6].on_screen = Some(false);+ case.nodes[6].offscreen_geometry = true;+ let parsed = items(&case);+ assert_eq!(parsed[0].body.as_deref(), Some("Visible article paragraph"));+ case.nodes[0].class_name = None;+ assert!(matches!(+ parse(&case).outcome,+ ValidatedParseOutcome::NotHandled+ ));+}++#[test]+fn webkit_gitlab_description_is_bounded_before_attributes_and_activity() {+ let mut case = cases().remove(2);+ case.nodes = vec![+ node(0, "AXGroup", "content", ""),+ node(1, "AXGroup", "", ""),+ node(2, "AXHeading", "gl-heading-1", "Synthetic issue"),+ node(2, "AXLink", "gl-avatar-link js-user-link", "Alice"),+ node(1, "AXHeading", "", "Summary"),+ node(1, "AXGroup", "", ""),+ node(2, "AXStaticText", "", "Restore the saved draft"),+ node(1, "AXButton", "", "Read more"),+ node(+ 1,+ "AXGroup",+ "gl-detail-layout-sidebar",+ "Sidebar attributes",+ ),+ node(1, "AXGroup", "system-note", "Bot changed milestone"),+ ];+ case.nodes[0].dom_identifier = Some("content-body".into());+ let parsed = items(&case);+ assert_eq!(parsed.len(), 2);+ assert_eq!(parsed[1].actor.as_deref(), Some("Alice"));+ assert_eq!(+ parsed[1].body.as_deref(),+ Some("Summary\nRestore the saved draft")+ );+ case.nodes[8].class_name = None;+ assert!(matches!(+ parse(&case).outcome,+ ValidatedParseOutcome::NotHandled+ ));+}++#[test]+fn native_table_axes_do_not_duplicate_rows_or_erase_authored_repetition() {+ let mut case = cases().remove(1);+ case.nodes = vec![+ node(0, "AXGroup", "timeline-comment", ""),+ node(1, "AXGroup", "markdown-body", ""),+ node(2, "AXTable", "", ""),+ node(3, "AXRow", "", ""),+ node(4, "AXCell", "", ""),+ node(5, "AXStaticText", "", "Repeated value"),+ node(3, "AXRow", "", ""),+ node(4, "AXCell", "", ""),+ node(5, "AXStaticText", "", "Repeated value"),+ node(3, "AXColumn", "", ""),+ node(4, "AXCell", "", ""),+ node(5, "AXStaticText", "", "Repeated value"),+ node(3, "AXGroup", "", ""),+ node(4, "AXCell", "", ""),+ node(5, "AXStaticText", "", "Repeated value"),+ ];+ assert_eq!(+ items(&case)[1].body.as_deref(),+ Some("Repeated value\nRepeated value")+ );+}++#[test]+fn native_role_labels_and_section_edit_controls_are_not_authored_text() {+ let mut case = cases().remove(1);+ case.nodes = vec![+ node(0, "AXGroup", "timeline-comment", ""),+ node(1, "AXGroup", "markdown-body", ""),+ node(2, "AXStaticText", "", ""),+ node(2, "AXLink", "", "edit"),+ node(3, "AXStaticText", "", "edit"),+ node(2, "AXStaticText", "", "text"),+ node(2, "AXStaticText", "", "edit"),+ ];+ case.nodes[2].role_description = Some("text".into());+ case.nodes[3].help_text = Some("Edit section: History".into());+ assert_eq!(items(&case)[1].body.as_deref(), Some("text\nedit"));+}++#[test]+fn windows_chromium_github_body_stays_inside_verified_post() {+ let mut case = cases().remove(1);+ case.app.platform = Platform::Windows;+ case.nodes = vec![+ node(+ 0,+ "Text",+ "PullRequestHeader-module__inlineTitle__synthetic",+ "Synthetic pull request",+ ),+ node(0, "Group", "timeline-comment", ""),+ node(1, "Hyperlink", "author", "Alice"),+ node(1, "Hyperlink", "js-timestamp", "Yesterday"),+ node(1, "Group", "edit-comment-hide", ""),+ node(2, "Group", "", ""),+ node(3, "Text", "", "Authored prefix"),+ node(4, "Hyperlink", "", "linked value"),+ node(3, "Text", "", "Authored suffix"),+ node(0, "Group", "TimelineItem-body", "Unrelated activity"),+ ];+ case.nodes[3].dom_identifier = Some("issue-42-permalink".into());+ let parsed = items(&case);+ assert_eq!(parsed[1].actor.as_deref(), Some("Alice"));+ assert_eq!(+ parsed[1].body.as_deref(),+ Some("Authored prefix\nlinked value\nAuthored suffix")+ );+ assert!(!parsed[1].body.as_ref().unwrap().contains("activity"));+}++#[test]+fn synthetic_windows_github_prefers_unclassified_body_and_structural_aggregate() {+ let mut case = cases().remove(1);+ case.app.platform = Platform::Windows;+ case.nodes = vec![+ node(+ 0,+ "Text",+ "PullRequestHeader-module__inlineTitle__synthetic",+ "Synthetic PR",+ ),+ node(0, "Group", "timeline-comment", ""),+ node(1, "Hyperlink", "author", "Alice"),+ node(1, "Hyperlink", "js-timestamp", "Yesterday"),+ node(1, "Group", "edit-comment-hide", ""),+ node(2, "Group", "toolbar", "Not authored"),+ node(2, "Group", "", ""),+ node(3, "Text", "", "Aggregate with linked value and suffix"),+ node(4, "Hyperlink", "", "linked value"),+ node(3, "Text", "", "Repeated paragraph"),+ node(3, "Text", "", "Repeated paragraph"),+ node(3, "Text", "", "code block\n indented line"),+ ];+ case.nodes[3].dom_identifier = Some("issue-42-permalink".into());+ let parsed = items(&case);+ assert_eq!(+ parsed[1].body.as_deref(),+ Some(+ "Aggregate with linked value and suffix\nRepeated paragraph\nRepeated paragraph\ncode block\n indented line"+ )+ );+}++#[test]+fn synthetic_windows_github_partial_parent_preserves_child_link_and_budget() {+ let mut case = cases().remove(1);+ case.app.platform = Platform::Windows;+ case.nodes = vec![+ node(+ 0,+ "Text",+ "PullRequestHeader-module__inlineTitle__synthetic",+ "Synthetic PR",+ ),+ node(0, "Group", "timeline-comment", ""),+ node(1, "Hyperlink", "author", "Alice"),+ node(1, "Hyperlink", "js-timestamp", "Yesterday"),+ node(1, "Group", "edit-comment-hide", ""),+ node(2, "Group", "", ""),+ node(3, "Text", "", "prefix"),+ node(4, "Hyperlink", "", "link"),+ node(3, "Text", "", "suffix"),+ node(3, "Button", "", "Copy"),+ node(3, "Text", "sidebar", "excluded"),+ ];+ case.nodes[3].dom_identifier = Some("issue-42-permalink".into());+ assert_eq!(+ items(&case)[1].body.as_deref(),+ Some("prefix\nlink\nsuffix")+ );++ case.nodes[8].text = "x".repeat(40_000);+ let parsed = items(&case);+ assert_eq!(+ parsed[1].metadata.get("truncated").map(String::as_str),+ Some("true")+ );+ assert!(parsed[1].body.as_ref().unwrap().len() <= 32 * 1024);+}++#[test]+fn windows_chromium_hn_requires_local_author_and_numeric_toggle() {+ let mut case = cases().remove(3);+ case.app.platform = Platform::Windows;+ case.nodes = vec![+ node(0, "Group", "title", ""),+ node(1, "Hyperlink", "", "Synthetic discussion"),+ node(0, "Group", "default", ""),+ node(1, "Hyperlink", "hnuser", "Alice"),+ node(1, "Hyperlink", "togg clicky", "[–]"),+ node(1, "Text", "", "Verified comment"),+ ];+ case.nodes[4].dom_identifier = Some("15".into());+ let parsed = items(&case);+ assert_eq!(parsed[1].actor.as_deref(), Some("Alice"));+ assert_eq!(parsed[1].body.as_deref(), Some("Verified comment"));+ case.nodes[4].dom_identifier = Some("not-a-native-comment-id".into());+ assert_eq!(parse(&case).outcome, ValidatedParseOutcome::NotHandled);+}++#[test]+fn synthetic_windows_hn_preserves_local_multifragment_body_only() {+ let mut case = cases().remove(3);+ case.app.platform = Platform::Windows;+ case.nodes = vec![+ node(0, "Group", "title", ""),+ node(1, "Hyperlink", "", "Synthetic discussion"),+ node(0, "Group", "default", ""),+ node(1, "Hyperlink", "hnuser", "Alice"),+ node(1, "Hyperlink", "togg clicky", "[–]"),+ node(1, "Text", "", "First paragraph"),+ node(1, "Hyperlink", "", "linked phrase"),+ node(1, "Text", "", "Second paragraph"),+ node(1, "Group", "default", ""),+ node(2, "Hyperlink", "hnuser", "Bob"),+ node(2, "Hyperlink", "togg", "[–]"),+ node(2, "Text", "", "Nested body"),+ ];+ case.nodes[4].dom_identifier = Some("15".into());+ case.nodes[10].dom_identifier = Some("16".into());+ let parsed = items(&case);+ assert_eq!(parsed[1].actor.as_deref(), Some("Alice"));+ assert_eq!(+ parsed[1].body.as_deref(),+ Some("First paragraph\nlinked phrase\nSecond paragraph")+ );+ assert!(!parsed[1].body.as_ref().unwrap().contains("Nested"));+}++#[test]+fn synthetic_windows_hn_rejects_missing_local_identity_evidence() {+ let mut case = cases().remove(3);+ case.app.platform = Platform::Windows;+ case.nodes = vec![+ node(0, "Group", "title", ""),+ node(1, "Hyperlink", "", "Synthetic discussion"),+ node(0, "Group", "default", ""),+ node(1, "Hyperlink", "togg", "[–]"),+ node(1, "Text", "", "Body without local author"),+ node(1, "Group", "default", ""),+ node(2, "Hyperlink", "hnuser", "Borrowed Bob"),+ node(2, "Hyperlink", "togg", "[–]"),+ node(2, "Text", "", "Nested body"),+ ];+ case.nodes[3].dom_identifier = Some("17".into());+ case.nodes[7].dom_identifier = Some("18".into());+ assert_eq!(parse(&case).outcome, ValidatedParseOutcome::NotHandled);++ case.nodes+ .insert(3, node(1, "Hyperlink", "hnuser", "Alice"));+ case.nodes[4].dom_identifier = Some(String::new());+ assert_eq!(parse(&case).outcome, ValidatedParseOutcome::NotHandled);+}++#[test]+fn windows_chromium_gitlab_creator_requires_by_label() {+ let mut case = cases().remove(2);+ case.app.platform = Platform::Windows;+ case.nodes = vec![+ node(0, "Group", "content", ""),+ node(1, "Group", "", ""),+ node(2, "Text", "gl-heading-1", "Synthetic work item"),+ node(2, "Text", "", "by"),+ node(2, "Hyperlink", "gl-avatar-link js-user-link", "Alice"),+ node(1, "Group", "", ""),+ node(2, "Group", "work-item-description", ""),+ node(3, "Text", "", "Verified description"),+ node(+ 3,+ "Hyperlink",+ "author-link js-user-link",+ "Later commenter",+ ),+ ];+ case.nodes[0].dom_identifier = Some("content-body".into());+ let parsed = items(&case);+ assert_eq!(parsed[1].actor.as_deref(), Some("Alice"));+ case.nodes[3].text = "near".into();+ assert_eq!(items(&case)[1].actor, None);+}