Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(linux): preserve semantic parser structure (#6458)

Louis Beaumont0d6c925f8688
diff --git a/.github/scripts/linux/bundle-appimage-runtime-deps.sh b/.github/scripts/linux/bundle-appimage-runtime-deps.sh
index 7a2eac23b..2c4fb1d28 100755
--- a/.github/scripts/linux/bundle-appimage-runtime-deps.sh
+++ b/.github/scripts/linux/bundle-appimage-runtime-deps.sh
@@ -278,6 +278,10 @@ done
# qwen3-asr links OpenBLAS on Linux. linuxdeploy can miss it when the library is
# pulled in through the build-time BLAS shim, so bundle the SONAME explicitly.
bundle_named_lib "libopenblas.so.0"
+# The app binary itself can retain a direct libgfortran dependency even when
+# the selected OpenBLAS build does not expose it transitively. A minimal Ubuntu
+# desktop does not install this runtime by default, so make it explicit too.
+bundle_named_lib "libgfortran.so.5"
# Generic Wayland capture connects to the desktop portal's PipeWire remote.
# linuxdeploy can miss this dependency because the capture path is lazy, but
diff --git a/.github/scripts/linux/test-bundle-appimage-runtime-deps.sh b/.github/scripts/linux/test-bundle-appimage-runtime-deps.sh
index d26a4195e..6c6bc439f 100755
--- a/.github/scripts/linux/test-bundle-appimage-runtime-deps.sh
+++ b/.github/scripts/linux/test-bundle-appimage-runtime-deps.sh
@@ -77,6 +77,7 @@ hook_before="$(sha256sum "${APPDIR}/apprun-hooks/linuxdeploy-existing-hook.sh")"
"${BUNDLER}" "${APPDIR}"
client="${APPDIR}/usr/lib/libpipewire-0.3.so.0"
+test -e "${APPDIR}/usr/lib/libgfortran.so.5"
config="${APPDIR}/usr/share/pipewire/client.conf"
module_dir="${APPDIR}/usr/lib/pipewire-0.3"
spa_dir="${APPDIR}/usr/lib/spa-0.2"
diff --git a/.github/workflows/linux-appimage-smoke.yml b/.github/workflows/linux-appimage-smoke.yml
index f727fa2d6..efecb497c 100644
--- a/.github/workflows/linux-appimage-smoke.yml
+++ b/.github/workflows/linux-appimage-smoke.yml
@@ -361,6 +361,10 @@ jobs:
echo "::error::libopenblas.so.0 is not bundled in the AppImage; OCR breaks on hosts without the system package"
exit 1
fi
+ if [ ! -e squashfs-root/usr/lib/libgfortran.so.5 ]; then
+ echo "::error::libgfortran.so.5 is not bundled in the AppImage; the app fails startup on minimal Linux hosts"
+ exit 1
+ fi
check_ldd squashfs-root/usr/lib/libopenblas.so.0 /tmp/openblas.ldd
if grep -R "not found" /tmp/ffmpeg.ldd /tmp/ffprobe.ldd /tmp/tesseract.ldd /tmp/openblas.ldd 2>/dev/null; then
exit 1
diff --git a/COVERAGE.md b/COVERAGE.md
index b4ac0d2ba..85a381bd6 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -32,15 +32,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 32
- Mapped Rust files: 327
-- Active test blocks: 3172
+- Active test blocks: 3175
- Ignored/manual test blocks: 137
-- Weighted coverage points: 2604.8
+- Weighted coverage points: 2606.2
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
| windows | 29 | 3036 | 132 | 2542.8 | 21 | 11 | 100% |
| macos | 29 | 3094 | 112 | 2555.2 | 22 | 11 | 100% |
-| linux | 25 | 2707 | 105 | 2244.6 | 20 | 11 | 100% |
+| linux | 25 | 2710 | 105 | 2246.1 | 20 | 11 | 100% |
## Refresh
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
index f06ed0da3..ae29cd8e8 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
@@ -1157,11 +1157,21 @@ async fn main() {
// reason parameter) is still the current Apple event.
let from_autostart = launched_from_autostart();
- #[cfg(any(windows, target_os = "linux"))]
+ #[cfg(windows)]
{
use tauri_plugin_deep_link::DeepLinkExt;
app.deep_link().register_all()?;
}
+ #[cfg(target_os = "linux")]
+ {
+ use tauri_plugin_deep_link::DeepLinkExt;
+ // Minimal Linux installs may not provide
+ // `update-desktop-database`. Deep-link registration is useful
+ // integration, but it must not make the recorder fail startup.
+ if let Err(error) = app.deep_link().register_all() {
+ eprintln!("deep-link registration skipped: {error}");
+ }
+ }
let app_handle = app.handle();
// The embedded and overlay native timelines can mount before the
diff --git a/crates/screenpipe-a11y/src/tree/linux.rs b/crates/screenpipe-a11y/src/tree/linux.rs
index c7d60501f..ed00a83d1 100644
--- a/crates/screenpipe-a11y/src/tree/linux.rs
+++ b/crates/screenpipe-a11y/src/tree/linux.rs
@@ -25,6 +25,7 @@ use anyhow::{Context, Result};
use chrono::Utc;
use screenpipe_core::window_pattern::{self, WindowPattern};
use std::cell::UnsafeCell;
+use std::collections::HashMap;
use std::time::Instant;
use tracing::{debug, warn};
use zbus::blocking::Connection;
@@ -75,6 +76,33 @@ fn is_automation_structure_role(role: u32) -> bool {
)
}
+/// AT-SPI containers that preserve parser-relevant hierarchy while semantic
+/// context is enabled. These nodes remain transient and are not added to the
+/// historical raw accessibility payload.
+fn is_document_role(role: u32) -> bool {
+ matches!(role, 82 | 94 | 95)
+}
+
+fn is_semantic_structure_role(role: u32) -> bool {
+ matches!(
+ role,
+ 16 // ROLE_DIALOG
+ | 23 // ROLE_FRAME
+ | 69 // ROLE_WINDOW
+ | 75 // ROLE_APPLICATION
+ | 20 // ROLE_FILLER
+ | 30 // ROLE_LIST
+ | 34 // ROLE_MENU_BAR
+ | 39 // ROLE_PANEL
+ | 55 // ROLE_TABLE
+ | 63 // ROLE_TOOL_BAR
+ | 85 // ROLE_SECTION
+ | 82 // ROLE_DOCUMENT_FRAME
+ | 94 // ROLE_DOCUMENT_TEXT
+ | 95 // ROLE_DOCUMENT_WEB
+ )
+}
+
/// Roles that carry user-visible text in their name, description, or Text interface.
/// Values from: https://docs.rs/atspi/latest/atspi/enum.Role.html
fn should_extract_text(role: u32) -> bool {
@@ -324,6 +352,24 @@ fn get_accessible_state(conn: &Connection, aref: &AccessibleRef) -> Vec<u32> {
.unwrap_or_default()
}
+/// Get implementation-specific AT-SPI attributes such as Chromium's HTML id
+/// and class hints. Unsupported toolkits return an empty map.
+fn get_accessible_attributes(conn: &Connection, aref: &AccessibleRef) -> HashMap<String, String> {
+ dbus_call(
+ conn,
+ &aref.bus_name,
+ &aref.path,
+ ATSPI_ACCESSIBLE,
+ "GetAttributes",
+ &(),
+ )
+ .and_then(|reply| {
+ let attributes: HashMap<String, String> = reply.body().deserialize()?;
+ Ok(attributes)
+ })
+ .unwrap_or_default()
+}
+
/// Get children of an accessible object.
/// Returns a list of (bus_name, object_path) pairs.
fn get_accessible_children(conn: &Connection, aref: &AccessibleRef) -> Vec<AccessibleRef> {
@@ -522,6 +568,8 @@ struct WalkState {
line_budget: Option<LineBudget>,
line_max_calls_per_node: usize,
line_min_height_ratio: f32,
+ /// Retain parser-relevant containers and DOM hints in memory only.
+ capture_semantic_structure: bool,
/// Retain unnamed interactive controls in the same bounded tree walk.
capture_automation_structure: bool,
}
@@ -565,6 +613,7 @@ impl WalkState {
},
line_max_calls_per_node: config.line_bounds_max_calls_per_node,
line_min_height_ratio: config.line_bounds_min_height_ratio,
+ capture_semantic_structure: config.capture_semantic_structure,
capture_automation_structure: config.capture_automation_structure,
}
}
@@ -603,7 +652,13 @@ impl WalkState {
}
/// Recursively walk the AT-SPI2 tree of a given accessible object.
-fn walk_accessible(conn: &Connection, aref: &AccessibleRef, depth: usize, state: &mut WalkState) {
+fn walk_accessible(
+ conn: &Connection,
+ aref: &AccessibleRef,
+ depth: usize,
+ inside_document: bool,
+ state: &mut WalkState,
+) {
if state.should_stop() || depth >= state.max_depth {
return;
}
@@ -618,6 +673,7 @@ fn walk_accessible(conn: &Connection, aref: &AccessibleRef, depth: usize, state:
Ok(r) => r,
Err(_) => return,
};
+ let inside_document = inside_document || is_document_role(role);
// Skip decorative roles
if should_skip_role(role)
@@ -640,11 +696,14 @@ fn walk_accessible(conn: &Connection, aref: &AccessibleRef, depth: usize, state:
}
}
- // Extract text from text-bearing elements
+ // Extract text from text-bearing elements. When semantic context is on,
+ // retain otherwise-omitted containers so the flat adapter can rebuild the
+ // actual web hierarchy instead of attaching content to stale controls.
+ let emitted_before = state.nodes.len();
if should_extract_text(role)
|| (state.capture_automation_structure && is_automation_structure_role(role))
{
- extract_text(conn, aref, role, depth, state);
+ extract_text(conn, aref, role, depth, inside_document, state);
} else if role == 39 /* Panel */ || role == 85 /* Section */ || role == 23
/* Frame */
{
@@ -654,6 +713,12 @@ fn walk_accessible(conn: &Connection, aref: &AccessibleRef, depth: usize, state:
// Only add short names for containers (long ones are usually content)
}
}
+ if state.capture_semantic_structure
+ && state.nodes.len() == emitted_before
+ && is_semantic_structure_role(role)
+ {
+ capture_semantic_structure(conn, aref, role, depth, inside_document, state);
+ }
if state.should_stop() {
return;
@@ -665,7 +730,7 @@ fn walk_accessible(conn: &Connection, aref: &AccessibleRef, depth: usize, state:
if state.should_stop() {
break;
}
- walk_accessible(conn, child, depth + 1, state);
+ walk_accessible(conn, child, depth + 1, inside_document, state);
}
}
@@ -680,14 +745,94 @@ fn fill_atspi_state_from_set(node: &mut AccessibilityTreeNode, state_set: &[u32]
}
}
-fn fill_atspi_state(node: &mut AccessibilityTreeNode, conn: &Connection, aref: &AccessibleRef) {
+fn first_nonempty_attribute(
+ attributes: &HashMap<String, String>,
+ names: &[&str],
+) -> Option<String> {
+ names.iter().find_map(|name| {
+ attributes
+ .get(*name)
+ .map(|value| value.trim())
+ .filter(|value| !value.is_empty())
+ .map(str::to_owned)
+ })
+}
+
+fn apply_atspi_attributes(
+ node: &mut AccessibilityTreeNode,
+ attributes: &HashMap<String, String>,
+ persist_automation_fields: bool,
+) {
+ let dom_identifier = first_nonempty_attribute(attributes, &["html-id", "id", "dom-id"]);
+ let dom_classes = first_nonempty_attribute(attributes, &["class", "class-name", "html-class"]);
+ node.semantic_dom_identifier = dom_identifier.clone();
+ node.semantic_dom_classes = dom_classes.clone();
+ node.placeholder = first_nonempty_attribute(
+ attributes,
+ &["placeholder-text", "placeholder", "html-placeholder"],
+ );
+ node.role_description =
+ first_nonempty_attribute(attributes, &["xml-roles", "role-description"]);
+ if persist_automation_fields {
+ node.class_name = dom_classes;
+ if node.automation_id.is_none() {
+ node.automation_id = dom_identifier;
+ }
+ }
+}
+
+fn finish_atspi_node(
+ node: &mut AccessibilityTreeNode,
+ conn: &Connection,
+ aref: &AccessibleRef,
+ role: u32,
+ inside_document: bool,
+ state: &WalkState,
+) {
let state_set = get_accessible_state(conn, aref);
fill_atspi_state_from_set(node, &state_set);
// Description doubles as help_text on Linux
let desc = get_accessible_description(conn, aref);
if !desc.is_empty() {
+ node.semantic_description = Some(desc.clone());
node.help_text = Some(desc);
}
+ node.walk_index = state.node_count.min(u32::MAX as usize) as u32;
+ node.automation_relevant =
+ state.capture_automation_structure && is_automation_structure_role(role);
+ if (state.capture_semantic_structure && inside_document) || state.capture_automation_structure {
+ let attributes = get_accessible_attributes(conn, aref);
+ apply_atspi_attributes(node, &attributes, state.capture_automation_structure);
+ }
+ if state.capture_automation_structure && node.automation_id.is_none() {
+ node.automation_id = Some(aref.path.clone());
+ }
+}
+
+fn capture_semantic_structure(
+ conn: &Connection,
+ aref: &AccessibleRef,
+ role: u32,
+ depth: usize,
+ inside_document: bool,
+ state: &mut WalkState,
+) {
+ let extents = get_component_extents(conn, aref);
+ let bounds = extents
+ .and_then(|(x, y, w, h)| normalize_bounds(x as f64, y as f64, w as f64, h as f64, state));
+ let on_screen = extents
+ .and_then(|(x, y, w, h)| is_on_screen(x as f64, y as f64, w as f64, h as f64, state));
+ let text = get_accessible_name(conn, aref);
+ let mut node = AccessibilityTreeNode::new(
+ role_name(role).to_owned(),
+ text.trim().to_owned(),
+ depth.min(255) as u8,
+ bounds,
+ );
+ node.on_screen = on_screen;
+ node.semantic_only = true;
+ finish_atspi_node(&mut node, conn, aref, role, inside_document, state);
+ state.nodes.push(node);
}
/// Extract text from a text-bearing accessible element.
@@ -696,6 +841,7 @@ fn extract_text(
aref: &AccessibleRef,
role: u32,
depth: usize,
+ inside_document: bool,
state: &mut WalkState,
) {
// Element extents in screen-absolute coords; bounds are normalized
@@ -723,8 +869,7 @@ fn extract_text(
);
node.on_screen = on_screen;
node.is_password = Some(true);
- node.automation_relevant = true;
- fill_atspi_state_from_set(&mut node, &state_set);
+ finish_atspi_node(&mut node, conn, aref, role, inside_document, state);
state.nodes.push(node);
}
return;
@@ -739,7 +884,7 @@ fn extract_text(
bounds.clone(),
);
node.on_screen = on_screen;
- fill_atspi_state(&mut node, conn, aref);
+ finish_atspi_node(&mut node, conn, aref, role, inside_document, state);
// Multi-line Entry / Text widgets (textareas, code editors) get
// per-line bounds so search highlights pinpoint the matched word.
// ROLE_TEXT (61) is the typical multi-line case; ROLE_ENTRY (79)
@@ -762,7 +907,7 @@ fn extract_text(
bounds.clone(),
);
node.on_screen = on_screen;
- fill_atspi_state(&mut node, conn, aref);
+ finish_atspi_node(&mut node, conn, aref, role, inside_document, state);
node.lines = capture_lines_for_node(conn, aref, &trimmed, &bounds, on_screen, state);
state.nodes.push(node);
return;
@@ -780,7 +925,7 @@ fn extract_text(
bounds,
);
node.on_screen = on_screen;
- fill_atspi_state(&mut node, conn, aref);
+ finish_atspi_node(&mut node, conn, aref, role, inside_document, state);
state.nodes.push(node);
return;
}
@@ -796,7 +941,7 @@ fn extract_text(
bounds,
);
node.on_screen = on_screen;
- fill_atspi_state(&mut node, conn, aref);
+ finish_atspi_node(&mut node, conn, aref, role, inside_document, state);
state.nodes.push(node);
return;
}
@@ -809,9 +954,7 @@ fn extract_text(
bounds,
);
node.on_screen = on_screen;
- node.automation_relevant = true;
- let state_set = get_accessible_state(conn, aref);
- fill_atspi_state_from_set(&mut node, &state_set);
+ finish_atspi_node(&mut node, conn, aref, role, inside_document, state);
state.nodes.push(node);
}
}
@@ -1182,7 +1325,7 @@ impl TreeWalkerPlatform for LinuxTreeWalker {
}
// Walk the accessibility tree
- walk_accessible(conn, &window_ref, 0, &mut state);
+ walk_accessible(conn, &window_ref, 0, false, &mut state);
if state.hit_ignored_extension {
debug!(
@@ -1236,6 +1379,11 @@ impl TreeWalkerPlatform for LinuxTreeWalker {
.then(|| crate::platform::linux::get_process_name(pid))
.flatten()
.or_else(|| self.config.capture_app_identity.then(|| app_name.clone()));
+ let (nodes, semantic_nodes) = split_structural_nodes(
+ std::mem::take(&mut state.nodes),
+ state.capture_semantic_structure,
+ state.capture_automation_structure,
+ );
Ok(TreeWalkResult::Found(TreeSnapshot {
app_name,
app_id: None,
@@ -1243,8 +1391,8 @@ impl TreeWalkerPlatform for LinuxTreeWalker {
app_version: None,
window_name: window_title,
text_content,
- nodes: state.nodes,
- semantic_nodes: Vec::new(),
+ nodes,
+ semantic_nodes,
browser_url,
document_path,
timestamp: Utc::now(),
@@ -1260,6 +1408,26 @@ impl TreeWalkerPlatform for LinuxTreeWalker {
}
}
+fn split_structural_nodes(
+ nodes: Vec<AccessibilityTreeNode>,
+ capture_semantic_structure: bool,
+ persist_automation_structure: bool,
+) -> (Vec<AccessibilityTreeNode>, Vec<AccessibilityTreeNode>) {
+ if persist_automation_structure {
+ if capture_semantic_structure {
+ nodes
+ .into_iter()
+ .partition(|node| !node.semantic_only || node.automation_relevant)
+ } else {
+ (nodes, Vec::new())
+ }
+ } else if capture_semantic_structure {
+ nodes.into_iter().partition(|node| !node.semantic_only)
+ } else {
+ (nodes, Vec::new())
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -1296,6 +1464,51 @@ mod tests {
assert!(!is_automation_structure_role(55)); // TABLE
}
+ #[test]
+ fn test_semantic_structure_roles_preserve_cross_document_topology() {
+ assert!(is_semantic_structure_role(95)); // DOCUMENT_WEB
+ assert!(is_semantic_structure_role(85)); // SECTION
+ assert!(is_semantic_structure_role(39)); // PANEL
+ assert!(!is_semantic_structure_role(43)); // PUSH_BUTTON
+ assert!(!is_semantic_structure_role(61)); // TEXT
+ }
+
+ #[test]
+ fn test_atspi_dom_hints_stay_transient_without_automation_mode() {
+ let mut attributes = HashMap::new();
+ attributes.insert("html-id".to_owned(), "message-42".to_owned());
+ attributes.insert("class".to_owned(), "font-user-message active".to_owned());
+ attributes.insert("placeholder-text".to_owned(), "Message".to_owned());
+ attributes.insert("xml-roles".to_owned(), "article".to_owned());
+ let mut node = AccessibilityTreeNode::new("Section".into(), String::new(), 1, None);
+
+ apply_atspi_attributes(&mut node, &attributes, false);
+
+ assert_eq!(node.semantic_dom_identifier.as_deref(), Some("message-42"));
+ assert_eq!(
+ node.semantic_dom_classes.as_deref(),
+ Some("font-user-message active")
+ );
+ assert_eq!(node.placeholder.as_deref(), Some("Message"));
+ assert_eq!(node.role_description.as_deref(), Some("article"));
+ assert_eq!(node.automation_id, None);
+ assert_eq!(node.class_name, None);
+ }
+
+ #[test]
+ fn test_split_structural_nodes_keeps_memory_only_shape_out_of_raw_tree() {
+ let raw = AccessibilityTreeNode::new("Static".into(), "visible".into(), 2, None);
+ let mut structural = AccessibilityTreeNode::new("Section".into(), String::new(), 1, None);
+ structural.semantic_only = true;
+
+ let (nodes, semantic_nodes) = split_structural_nodes(vec![structural, raw], true, false);
+
+ assert_eq!(nodes.len(), 1);
+ assert_eq!(nodes[0].text, "visible");
+ assert_eq!(semantic_nodes.len(), 1);
+ assert_eq!(semantic_nodes[0].role, "Section");
+ }
+
#[test]
fn test_role_name() {
assert_eq!(role_name(43), "Button");
diff --git a/crates/screenpipe-semantic/src/parsers/chatgpt.rs b/crates/screenpipe-semantic/src/parsers/chatgpt.rs
index d1f65b111..53a89dc61 100644
--- a/crates/screenpipe-semantic/src/parsers/chatgpt.rs
+++ b/crates/screenpipe-semantic/src/parsers/chatgpt.rs
@@ -25,7 +25,7 @@ impl ChatGptParser {
Self {
manifest: ParserManifest {
id: "app.chatgpt.turn_markers".into(),
- parser_version: "2".into(),
+ parser_version: "3".into(),
schema_version: 1,
scope: ParserScope::App,
platforms: vec![Platform::Macos, Platform::Windows, Platform::Linux],
@@ -158,6 +158,7 @@ struct Turn {
struct PendingTurn {
marker: NodeId,
actor: &'static str,
+ document_scope: Option<NodeId>,
body: String,
last_line: Option<String>,
}
@@ -177,6 +178,7 @@ fn actor_delimited_turns(tree: &SemanticTree) -> Vec<Turn> {
pending = Some(PendingTurn {
marker: node,
actor,
+ document_scope: document_scope(tree, node),
body: String::new(),
last_line: None,
});
@@ -186,6 +188,9 @@ fn actor_delimited_turns(tree: &SemanticTree) -> Vec<Turn> {
let Some(turn) = pending.as_mut() else {
continue;
};
+ if turn.document_scope.is_some() && document_scope(tree, node) != turn.document_scope {
+ continue;
+ }
if is_actor_label(node_content(tree, node))
|| !is_message_text_role(tree.role(node))
|| inside_ignored_control(tree, node)
@@ -226,6 +231,7 @@ fn windows_actor_delimited_turns(tree: &SemanticTree) -> Vec<Turn> {
pending = Some(PendingTurn {
marker: node,
actor,
+ document_scope: None,
body: String::new(),
last_line: None,
});
@@ -429,6 +435,9 @@ fn is_message_text_role(role: Option<&str>) -> bool {
fn inside_ignored_control(tree: &SemanticTree, node: NodeId) -> bool {
let mut current = Some(node);
while let Some(candidate) = current {
+ if is_document_role(tree.role(candidate)) {
+ return false;
+ }
if tree.role(candidate).is_some_and(|role| {
[
"AXButton",
@@ -449,6 +458,25 @@ fn inside_ignored_control(tree: &SemanticTree, node: NodeId) -> bool {
false
}
+fn is_document_role(role: Option<&str>) -> bool {
+ role.is_some_and(|role| {
+ ["AXWebArea", "Document", "DocumentFrame", "DocumentWeb"]
+ .iter()
+ .any(|document| document.eq_ignore_ascii_case(role))
+ })
+}
+
+fn document_scope(tree: &SemanticTree, node: NodeId) -> Option<NodeId> {
+ let mut current = Some(node);
+ while let Some(candidate) = current {
+ if is_document_role(tree.role(candidate)) {
+ return Some(candidate);
+ }
+ current = tree.parent(candidate);
+ }
+ None
+}
+
fn node_content(tree: &SemanticTree, node: NodeId) -> Option<&str> {
tree.value(node)
.or_else(|| tree.text(node))
@@ -607,6 +635,90 @@ mod tests {
assert_eq!(items[2].body.as_deref(), Some("Use a bounded replay eval."));
}
+ #[test]
+ fn linux_document_scope_drops_browser_chrome_after_last_turn() {
+ let mut builder = SemanticTreeBuilder::new(TreeBudget::default());
+ let window = builder
+ .push(None, input("Frame", Some("Linux Chat Parser Lab")))
+ .unwrap();
+ // Retained flat captures can still attach the web document to the
+ // nearest browser-chrome control when intermediate panels are omitted.
+ let browser_button = builder
+ .push(Some(window), input("Button", Some("Browser chrome")))
+ .unwrap();
+ let document = builder
+ .push(
+ Some(browser_button),
+ input("DocumentWeb", Some("Linux Chat Parser Lab")),
+ )
+ .unwrap();
+ let user_section = builder
+ .push(Some(document), input("Section", None))
+ .unwrap();
+ builder
+ .push(Some(user_section), input("Heading", Some("You said")))
+ .unwrap();
+ builder
+ .push(
+ Some(user_section),
+ input("Static", Some("hello from synthetic Ubuntu")),
+ )
+ .unwrap();
+ let assistant_section = builder
+ .push(Some(document), input("Section", None))
+ .unwrap();
+ builder
+ .push(
+ Some(assistant_section),
+ input("Heading", Some("ChatGPT said")),
+ )
+ .unwrap();
+ builder
+ .push(
+ Some(assistant_section),
+ input("Static", Some("safe assistant reply")),
+ )
+ .unwrap();
+ builder
+ .push(
+ Some(window),
+ input("Static", Some("data:text/html,<browser chrome>")),
+ )
+ .unwrap();
+ builder
+ .push(Some(window), input("Static", Some("Create split view")))
+ .unwrap();
+
+ let tree = builder.finish();
+ let app = AppIdentity {
+ platform: Platform::Linux,
+ app_id: None,
+ executable: Some("google-chrome".into()),
+ display_name: "Google Chrome".into(),
+ version: None,
+ browser_url: Some("https://chatgpt.com/c/synthetic".into()),
+ };
+ let context = ParseContext {
+ frame_id: 1,
+ captured_at_unix_ms: 2,
+ utc_offset_minutes: None,
+ locale_hint: None,
+ app: &app,
+ input_content_hash: 3,
+ };
+ let ParseOutcome::Handled(items) = ChatGptParser::new().parse(&context, &tree).unwrap()
+ else {
+ panic!("expected handled projection");
+ };
+
+ assert_eq!(items.len(), 3);
+ assert_eq!(
+ items[1].body.as_deref(),
+ Some("hello from synthetic Ubuntu")
+ );
+ assert_eq!(items[2].body.as_deref(), Some("safe assistant reply"));
+ }
+
#[test]
fn abstains_without_explicit_actor_headings() {
let mut builder = SemanticTreeBuilder::new(TreeBudget::default());
diff --git a/crates/screenpipe-semantic/src/parsers/obsidian.rs b/crates/screenpipe-semantic/src/parsers/obsidian.rs
index 7f072ab56..97aabdd9d 100644
--- a/crates/screenpipe-semantic/src/parsers/obsidian.rs
+++ b/crates/screenpipe-semantic/src/parsers/obsidian.rs
@@ -22,7 +22,7 @@ impl ObsidianParser {
Self {
manifest: ParserManifest {
id: "app.obsidian.active_note".into(),
- parser_version: "1".into(),
+ parser_version: "2".into(),
schema_version: 1,
scope: ParserScope::App,
platforms: vec![Platform::Macos, Platform::Windows, Platform::Linux],
@@ -70,6 +70,12 @@ impl SemanticParser for ObsidianParser {
let source = find_below_or_all(tree, search_root, |tree, node| {
role_is(tree, node, &["AXTextArea", "Edit"]) && has_class(tree, node, "cm-content")
});
+ let source_lines = find_below_or_all(tree, search_root, |tree, node| {
+ has_class(tree, node, "markdown-source-view")
+ && tree
+ .descendants(node)
+ .any(|descendant| has_class(tree, descendant, "cm-line"))
+ });
let preview = find_below_or_all(tree, search_root, |tree, node| {
has_class(tree, node, "markdown-preview-view")
|| has_class(tree, node, "markdown-rendered")
@@ -80,6 +86,11 @@ impl SemanticParser for ObsidianParser {
return Ok(ParseOutcome::NotHandled);
};
(node, body.to_owned(), "codemirror_source")
+ } else if let Some(node) = source_lines {
+ let Some(body) = collect_codemirror_lines(tree, node) else {
+ return Ok(ParseOutcome::NotHandled);
+ };
+ (node, body, "codemirror_lines")
} else if let Some(node) = preview {
let Some(body) = collect_text(tree, node) else {
return Ok(ParseOutcome::NotHandled);
@@ -108,6 +119,52 @@ impl SemanticParser for ObsidianParser {
}
}
+fn collect_codemirror_lines(tree: &SemanticTree, root: NodeId) -> Option<String> {
+ let mut lines = Vec::<String>::new();
+ for line_node in tree
+ .descendants(root)
+ .filter(|node| has_class(tree, *node, "cm-line"))
+ .take(2_000)
+ {
+ let mut fragments = Vec::<&str>::new();
+ for node in tree.descendants(line_node) {
+ if !is_text_role(tree.role(node)) || inside_control(tree, node, line_node) {
+ continue;
+ }
+ let Some(content) = node_content(tree, node) else {
+ continue;
+ };
+ let content = content.trim_matches(|character: char| {
+ character.is_whitespace() || matches!(character, '\u{200b}' | '\u{feff}')
+ });
+ if content.is_empty()
+ || fragments
+ .last()
+ .is_some_and(|previous| *previous == content)
+ {
+ continue;
+ }
+ fragments.push(content);
+ }
+ let line = fragments.join(" ");
+ if line.is_empty() {
+ if !lines.is_empty() && lines.last().is_some_and(|previous| !previous.is_empty()) {
+ lines.push(String::new());
+ }
+ } else {
+ lines.push(line);
+ }
+ }
+ while lines.last().is_some_and(String::is_empty) {
+ lines.pop();
+ }
+ if lines.is_empty() {
+ return None;
+ }
+ let body = lines.join("\n");
+ Some(truncate_str(&body, MAX_BODY_BYTES).to_owned())
+}
+
fn historical_text_area(tree: &SemanticTree) -> Option<(NodeId, &str)> {
all_nodes(tree)
.filter(|node| role_is(tree, *node, &["AXTextArea", "Edit"]))
diff --git a/crates/screenpipe-semantic/tests/claude_obsidian_parsers.rs b/crates/screenpipe-semantic/tests/claude_obsidian_parsers.rs
index 3f8662b1c..ec73cd09d 100644
--- a/crates/screenpipe-semantic/tests/claude_obsidian_parsers.rs
+++ b/crates/screenpipe-semantic/tests/claude_obsidian_parsers.rs
@@ -181,6 +181,26 @@ fn obsidian_extracts_the_active_codemirror_note() {
);
}
+#[test]
+fn obsidian_extracts_linux_codemirror_lines() {
+ let items = handled(
+ include_str!("fixtures/apps/obsidian_linux_codemirror.json"),
+ "app.obsidian.active_note",
+ );
+ assert_eq!(items.len(), 1);
+ assert_eq!(items[0].title.as_deref(), Some("Linux Parser Note"));
+ assert_eq!(
+ items[0].body.as_deref(),
+ Some(
+ "# Linux Parser Note\n\nSynthetic Obsidian content for the screenpipe Linux parser VM.\n\n- accessibility hierarchy\n- active note body\n- safe local fixture"
+ )
+ );
+ assert_eq!(
+ items[0].metadata.get("surface").map(String::as_str),
+ Some("codemirror_lines")
+ );
+}
+
#[test]
fn obsidian_extracts_the_rendered_preview() {
let items = handled(
diff --git a/crates/screenpipe-semantic/tests/fixtures/apps/obsidian_linux_codemirror.json b/crates/screenpipe-semantic/tests/fixtures/apps/obsidian_linux_codemirror.json
new file mode 100644
index 000000000..ac74227bd
--- /dev/null
+++ b/crates/screenpipe-semantic/tests/fixtures/apps/obsidian_linux_codemirror.json
@@ -0,0 +1,37 @@
+{
+ "app": {
+ "platform": "linux",
+ "app_id": null,
+ "executable": "Obsidian",
+ "display_name": "Obsidian",
+ "version": "1.13.7",
+ "browser_url": null
+ },
+ "nodes": [
+ { "parent": null, "role": "Frame", "text": "Linux Parser Note - Obsidian" },
+ { "parent": 0, "role": "DocumentWeb", "text": "Linux Parser Note - Obsidian" },
+ { "parent": 1, "role": "Section", "classes": ["workspace-tabs", "mod-active"] },
+ { "parent": 2, "role": "Section", "classes": ["workspace-leaf", "mod-active"] },
+ { "parent": 3, "role": "Section", "classes": ["view-header-title"] },
+ { "parent": 4, "role": "Static", "text": "Linux Parser Note" },
+ { "parent": 3, "role": "Section", "classes": ["markdown-source-view", "mod-cm6", "is-live-preview"] },
+ { "parent": 6, "role": "Section", "classes": ["cm-line", "HyperMD-header-1"] },
+ { "parent": 7, "role": "Static", "text": "#" },
+ { "parent": 7, "role": "Static", "text": "Linux Parser Note" },
+ { "parent": 6, "role": "Section", "classes": ["cm-line"] },
+ { "parent": 10, "role": "Static", "text": "" },
+ { "parent": 6, "role": "Section", "classes": ["cm-line"] },
+ { "parent": 12, "role": "Static", "text": "Synthetic Obsidian content for the screenpipe Linux parser VM." },
+ { "parent": 6, "role": "Section", "classes": ["cm-line"] },
+ { "parent": 14, "role": "Static", "text": "\u200b" },
+ { "parent": 6, "role": "Section", "classes": ["cm-line", "HyperMD-list-line-1"] },
+ { "parent": 16, "role": "Static", "text": "-" },
+ { "parent": 16, "role": "Static", "text": "accessibility hierarchy" },
+ { "parent": 6, "role": "Section", "classes": ["cm-line", "HyperMD-list-line-1"] },
+ { "parent": 19, "role": "Static", "text": "-" },
+ { "parent": 19, "role": "Static", "text": "active note body" },
+ { "parent": 6, "role": "Section", "classes": ["cm-line", "HyperMD-list-line-1"] },
+ { "parent": 22, "role": "Static", "text": "-" },
+ { "parent": 22, "role": "Static", "text": "safe local fixture" }
+ ]
+}
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 38120b03c..25e59a21c 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y
- Mapped suites: 32
- Mapped Rust files: 327
-- Active test blocks: 3172
+- Active test blocks: 3175
- Ignored/manual test blocks: 137
-- Declared test blocks: 3309
-- Weighted coverage points: 2604.8
+- Declared test blocks: 3312
+- Weighted coverage points: 2606.2
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -25,7 +25,7 @@ are explicitly enabled in a runtime lane.
| --- | --- | --- | --- | --- | --- | --- | --- |
| windows | 29 | 3036 | 132 | 2542.8 | 21 | 11 | 100% |
| macos | 29 | 3094 | 112 | 2555.2 | 22 | 11 | 100% |
-| linux | 25 | 2707 | 105 | 2244.6 | 20 | 11 | 100% |
+| linux | 25 | 2710 | 105 | 2246.1 | 20 | 11 | 100% |
## Crate Summary
@@ -36,7 +36,7 @@ are explicitly enabled in a runtime lane.
| screenpipe-sqlite-coordinator | 1 | 0 | 2 | 16 | 0 | 16.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 591 | 43 | 517.4 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 252 | 9 | 226.1 | 4 |
-| screenpipe-a11y | 4 | 2 | 28 | 340 | 27 | 251.3 | 3 |
+| screenpipe-a11y | 4 | 2 | 28 | 343 | 27 | 252.7 | 3 |
## Line Coverage
@@ -61,7 +61,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| Layer | windows | macos | linux |
| --- | --- | --- | --- |
-| accessibility | 4 suites / 345 active / 29 ignored / 316.5 pts | 4 suites / 399 active / 9 ignored / 323.4 pts | 4 suites / 320 active / 7 ignored / 294.0 pts |
+| accessibility | 4 suites / 345 active / 29 ignored / 316.5 pts | 4 suites / 399 active / 9 ignored / 323.4 pts | 4 suites / 323 active / 7 ignored / 295.5 pts |
| audio | 7 suites / 701 active / 44 ignored / 627.4 pts | 7 suites / 701 active / 44 ignored / 627.4 pts | 6 suites / 626 active / 43 ignored / 574.9 pts |
| audio-device | 2 suites / 212 active / 7 ignored / 189.5 pts | 2 suites / 212 active / 7 ignored / 189.5 pts | 1 suites / 137 active / 6 ignored / 137.0 pts |
| configuration | 2 suites / 138 active / 3 ignored / 124.2 pts | 2 suites / 138 active / 3 ignored / 124.2 pts | 2 suites / 138 active / 3 ignored / 124.2 pts |
@@ -74,7 +74,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
| performance | 13 suites / 1416 active / 67 ignored / 1244.9 pts | 14 suites / 1519 active / 70 ignored / 1286.1 pts | 13 suites / 1416 active / 67 ignored / 1244.9 pts |
| pipes | 1 suites / 466 active / 3 ignored / 326.2 pts | 1 suites / 466 active / 3 ignored / 326.2 pts | 1 suites / 466 active / 3 ignored / 326.2 pts |
-| privacy | 5 suites / 883 active / 36 ignored / 718.3 pts | 5 suites / 937 active / 16 ignored / 725.2 pts | 5 suites / 858 active / 14 ignored / 695.8 pts |
+| privacy | 5 suites / 883 active / 36 ignored / 718.3 pts | 5 suites / 937 active / 16 ignored / 725.2 pts | 5 suites / 861 active / 14 ignored / 697.2 pts |
| real-app | - | 1 suites / 103 active / 3 ignored / 41.2 pts | - |
| speaker | 2 suites / 348 active / 8 ignored / 348.0 pts | 2 suites / 348 active / 8 ignored / 348.0 pts | 2 suites / 348 active / 8 ignored / 348.0 pts |
| storage | 3 suites / 507 active / 29 ignored / 408.6 pts | 3 suites / 507 active / 29 ignored / 408.6 pts | 3 suites / 507 active / 29 ignored / 408.6 pts |
@@ -119,7 +119,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| Suite | Crate | Platforms | Layers | Flows | Criticality | Confidence | Kind | Files | Active | Ignored | Notes |
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
| a11y-core-tree-cross-platform | screenpipe-a11y | windows, macos, linux | accessibility, ui-events, privacy, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | strong | unit | 14 | 164 | 0 | Cross-platform accessibility config, tree normalization, cache, privacy title matching, events, budget, and activity feed units. |
-| a11y-linux-tree | screenpipe-a11y | linux | accessibility, privacy | accessibility-ui-events, privacy-and-redaction | medium | partial | unit | 4 | 24 | 1 | Linux-specific accessibility/incognito normalization tests. |
+| a11y-linux-tree | screenpipe-a11y | linux | accessibility, privacy | accessibility-ui-events, privacy-and-redaction | medium | partial | unit | 4 | 27 | 1 | Linux-specific accessibility/incognito normalization tests. |
| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 6 | 103 | 3 | macOS AX unit coverage plus real TextEdit/Finder/Obsidian probes. Obsidian tests are ignored by default because they require a local app install and AX permission. |
| a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 6 | 49 | 23 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. |
| audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 137 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). |
@@ -176,7 +176,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| a11y-core-tree-cross-platform | screenpipe-a11y | src/tree/electron_docs.rs | source | 17 | 0 | 17 |
| a11y-core-tree-cross-platform | screenpipe-a11y | src/tree/enhanced_mode_cache.rs | source | 15 | 0 | 15 |
| a11y-linux-tree | screenpipe-a11y | src/tree/linux_lines.rs | source | 3 | 0 | 3 |
-| a11y-linux-tree | screenpipe-a11y | src/tree/linux.rs | source | 10 | 0 | 10 |
+| a11y-linux-tree | screenpipe-a11y | src/tree/linux.rs | source | 13 | 0 | 13 |
| a11y-macos-tree | screenpipe-a11y | src/tree/macos_lines.rs | source | 12 | 0 | 12 |
| a11y-macos-tree | screenpipe-a11y | src/tree/macos.rs | source | 51 | 0 | 51 |
| a11y-core-tree-cross-platform | screenpipe-a11y | src/tree/mod.rs | source | 36 | 0 | 36 |