Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

feat(semantic): extract content from Windows UIA surfaces

Louis Beaumont9588e3693596
diff --git a/crates/screenpipe-semantic/src/parsers/catalog.rs b/crates/screenpipe-semantic/src/parsers/catalog.rs
index 183cd2c00..d478924ab 100644
--- a/crates/screenpipe-semantic/src/parsers/catalog.rs
+++ b/crates/screenpipe-semantic/src/parsers/catalog.rs
@@ -20,7 +20,7 @@ pub enum AppFamily {
/// Public app identity and family membership for one built-in parser profile.
///
-/// The catalog describes 47 supported app targets using public app identities,
+/// The catalog describes 49 supported app targets using public app identities,
/// URL patterns, and stable accessibility contracts.
#[derive(Debug, Clone, Copy)]
pub struct BuiltinAppProfile {
@@ -163,6 +163,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[
&["app.clickup.com/"],
Some("https://app.clickup.com/123/v/l/456"),
),
+ profile(
+ "codex",
+ "Codex",
+ &[C],
+ &[],
+ &["Codex", "Codex.exe", "codex"],
+ &[],
+ &[],
+ None,
+ ),
profile(
"cursor",
"Cursor",
@@ -521,6 +531,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[
&["web.whatsapp.com/"],
Some("https://web.whatsapp.com/"),
),
+ profile(
+ "windowsterminal",
+ "Windows Terminal",
+ &[R],
+ &[],
+ &["WindowsTerminal.exe", "wt", "wt.exe"],
+ &[],
+ &[],
+ None,
+ ),
profile(
"windsurf",
"Windsurf",
diff --git a/crates/screenpipe-semantic/src/parsers/chatgpt.rs b/crates/screenpipe-semantic/src/parsers/chatgpt.rs
index d1931c3bf..b5f5469ec 100644
--- a/crates/screenpipe-semantic/src/parsers/chatgpt.rs
+++ b/crates/screenpipe-semantic/src/parsers/chatgpt.rs
@@ -59,10 +59,18 @@ impl SemanticParser for ChatGptParser {
fn parse(
&self,
- _context: &ParseContext<'_>,
+ context: &ParseContext<'_>,
tree: &SemanticTree,
) -> Result<ParseOutcome, ProjectionError> {
- let turns = actor_delimited_turns(tree);
+ let mut turns = actor_delimited_turns(tree);
+ if turns.is_empty() && context.app.platform == Platform::Windows {
+ turns = windows_actor_delimited_turns(tree);
+ if turns.is_empty() && windows_landing_surface(tree) {
+ // Recognized ChatGPT landing/new-chat surface: zero
+ // conversation records is the truthful projection.
+ return Ok(ParseOutcome::Empty);
+ }
+ }
if turns.is_empty() {
return Ok(ParseOutcome::NotHandled);
}
@@ -159,6 +167,127 @@ fn actor_delimited_turns(tree: &SemanticTree) -> Vec<Turn> {
turns
}
+/// Windows Chromium-UIA variant of `actor_delimited_turns`. The persisted
+/// tree keeps only text-bearing leaves: heading-ness survives solely as
+/// role_description "heading" (mapped into `description`), adapter
+/// reparenting gives every web leaf a browser-chrome Button ancestor (so the
+/// ancestor-based control filter cannot be reused), and ChatGPT's own
+/// `wm-*` classes mark sidebar/composer/app chrome to exclude per node.
+fn windows_actor_delimited_turns(tree: &SemanticTree) -> Vec<Turn> {
+ let mut turns = Vec::with_capacity(8);
+ let mut pending: Option<PendingTurn> = None;
+ let mut retained_bytes = 0usize;
+
+ 'roots: for root in tree.roots() {
+ for node in tree.descendants(root) {
+ if let Some(actor) = windows_actor_heading(tree, node) {
+ finish_turn(&mut turns, pending.take());
+ if turns.len() == MAX_MESSAGES || retained_bytes == MAX_BODY_BYTES {
+ break 'roots;
+ }
+ pending = Some(PendingTurn {
+ marker: node,
+ actor,
+ body: String::new(),
+ last_line: None,
+ });
+ continue;
+ }
+
+ let Some(turn) = pending.as_mut() else {
+ continue;
+ };
+ if is_actor_label(windows_node_content(tree, node))
+ || !is_windows_body_role(tree.role(node))
+ || has_excluded_class(tree, node)
+ {
+ continue;
+ }
+ let Some(content) = windows_node_content(tree, node) else {
+ continue;
+ };
+ append_content(turn, content, &mut retained_bytes);
+ if retained_bytes == MAX_BODY_BYTES {
+ break 'roots;
+ }
+ }
+ }
+ finish_turn(&mut turns, pending);
+ turns
+}
+
+fn windows_actor_heading(tree: &SemanticTree, node: NodeId) -> Option<&'static str> {
+ if !is_windows_heading(tree, node) {
+ return None;
+ }
+ actor_for_label(windows_node_content(tree, node)?)
+}
+
+fn is_windows_heading(tree: &SemanticTree, node: NodeId) -> bool {
+ tree.role(node)
+ .is_some_and(|role| role.eq_ignore_ascii_case("Text"))
+ && tree
+ .description(node)
+ .is_some_and(|description| description.eq_ignore_ascii_case("heading"))
+}
+
+fn is_windows_body_role(role: Option<&str>) -> bool {
+ role.is_some_and(|role| {
+ ["Text", "Hyperlink", "ListItem"]
+ .iter()
+ .any(|candidate| candidate.eq_ignore_ascii_case(role))
+ })
+}
+
+const EXCLUDED_CLASS_PREFIXES: [&str; 3] = ["wm-sidebar", "wm-composer", "wm-app"];
+
+fn has_excluded_class(tree: &SemanticTree, node: NodeId) -> bool {
+ tree.classes(node).any(|class| {
+ EXCLUDED_CLASS_PREFIXES.iter().any(|prefix| {
+ class
+ .get(..prefix.len())
+ .is_some_and(|head| head.eq_ignore_ascii_case(prefix))
+ })
+ })
+}
+
+/// On Windows, `Hyperlink.value` is the href URL while the visible anchor
+/// text lives in `text` — the inverse of the macOS value-first priority.
+fn windows_node_content(tree: &SemanticTree, node: NodeId) -> Option<&str> {
+ tree.text(node)
+ .or_else(|| tree.value(node))
+ .or_else(|| tree.description(node))
+ .map(str::trim)
+ .filter(|content| !content.is_empty())
+}
+
+/// ChatGPT's landing/new-chat surface: requires BOTH the composer Edit and
+/// the landing heading class, so a conversation page whose turn markup
+/// drifted still returns NotHandled instead of a false Empty.
+fn windows_landing_surface(tree: &SemanticTree) -> bool {
+ let mut composer = false;
+ let mut landing_heading = false;
+ for root in tree.roots() {
+ for node in tree.descendants(root) {
+ if tree
+ .role(node)
+ .is_some_and(|role| role.eq_ignore_ascii_case("Edit"))
+ && (tree.identifier(node) == Some("mobile-composer-prompt")
+ || tree.has_class(node, "wm-composer-textarea"))
+ {
+ composer = true;
+ }
+ if tree.has_class(node, "wm-app-landingHeading") {
+ landing_heading = true;
+ }
+ if composer && landing_heading {
+ return true;
+ }
+ }
+ }
+ false
+}
+
fn finish_turn(turns: &mut Vec<Turn>, pending: Option<PendingTurn>) {
let Some(pending) = pending else {
return;
diff --git a/crates/screenpipe-semantic/src/parsers/claude.rs b/crates/screenpipe-semantic/src/parsers/claude.rs
index 242dfc78d..d9a6c2eb0 100644
--- a/crates/screenpipe-semantic/src/parsers/claude.rs
+++ b/crates/screenpipe-semantic/src/parsers/claude.rs
@@ -21,6 +21,14 @@ const IGNORED_ACTIONS: &[&str] = &[
"More actions",
"Share",
];
+// Windows Electron surface: the desktop app announces each turn through
+// sr-only live-region Text nodes. The user announcement carries the complete
+// message; the assistant announcement is a truncated prefix upgraded from the
+// visible body below it.
+const USER_ANNOUNCEMENT: &str = "You said:";
+const ASSISTANT_ANNOUNCEMENT: &str = "Claude responded:";
+const TURN_STOP_CLASSES: &[&str] = &["group/btn", "group/dd", "cds-reset"];
+const CHROME_LINES: &[&str] = &["Chat mode", "Fast mode off", "Show message actions"];
/// Exact Claude conversation parser. It requires actor-bearing DOM markers and
/// abstains on older or incompatible layouts so the shared conversation and
@@ -71,44 +79,34 @@ impl SemanticParser for ClaudeParser {
fn parse(
&self,
- _context: &ParseContext<'_>,
+ context: &ParseContext<'_>,
tree: &SemanticTree,
) -> Result<ParseOutcome, ProjectionError> {
- let mut messages = Vec::new();
- let mut retained_bytes = 0usize;
- for node in all_nodes(tree) {
- let Some(actor) = actor_for_node(tree, node) else {
- continue;
- };
- if marker_ancestor(tree, node).is_some() {
- continue;
- }
- let Some(body) = collect_text(tree, node, &mut retained_bytes) else {
- continue;
- };
- messages.push((node, actor, body));
- if messages.len() == MAX_MESSAGES || retained_bytes == MAX_BODY_BYTES {
- break;
- }
- }
- let has_user = messages.iter().any(|(_, actor, _)| *actor == "[user]");
- let has_assistant = messages.iter().any(|(_, actor, _)| *actor == "Claude");
- if !has_user || !has_assistant {
+ let (messages, surface, key_from_url) = if let Some(messages) = dom_marker_messages(tree) {
+ (messages, "actor_dom_markers", false)
+ } else if let Some(messages) = announcement_messages(tree) {
+ (messages, "sr_announcements", true)
+ } else {
return Ok(ParseOutcome::NotHandled);
- }
+ };
- let title = first_root_title(tree).unwrap_or("Claude");
+ let title = first_root_title_excluding(tree, "WinCaptionButton").unwrap_or("Claude");
+ let key_seed = if key_from_url {
+ context.app.browser_url.as_deref().unwrap_or(title)
+ } else {
+ title
+ };
let mut conversation = SemanticItem::new(
"conversation",
SemanticKind::Conversation,
- format!("claude:conversation:{}", key_component(title)),
+ format!("claude:conversation:{}", key_component(key_seed)),
IdentityQuality::Derived,
);
conversation.title = Some(title.to_owned());
conversation.metadata.insert("app".into(), "Claude".into());
conversation
.metadata
- .insert("surface".into(), "actor_dom_markers".into());
+ .insert("surface".into(), surface.into());
conversation.source_nodes.push(messages[0].0);
let mut items = Vec::with_capacity(messages.len() + 1);
@@ -130,6 +128,182 @@ impl SemanticParser for ClaudeParser {
}
}
+/// Web/macOS surface: actor-bearing DOM marker classes. Unchanged behavior.
+fn dom_marker_messages(tree: &SemanticTree) -> Option<Vec<(NodeId, &'static str, String)>> {
+ let mut messages = Vec::new();
+ let mut retained_bytes = 0usize;
+ for node in all_nodes(tree) {
+ let Some(actor) = actor_for_node(tree, node) else {
+ continue;
+ };
+ if marker_ancestor(tree, node).is_some() {
+ continue;
+ }
+ let Some(body) = collect_text(tree, node, &mut retained_bytes) else {
+ continue;
+ };
+ messages.push((node, actor, body));
+ if messages.len() == MAX_MESSAGES || retained_bytes == MAX_BODY_BYTES {
+ break;
+ }
+ }
+ let has_user = messages.iter().any(|(_, actor, _)| *actor == "[user]");
+ let has_assistant = messages.iter().any(|(_, actor, _)| *actor == "Claude");
+ (has_user && has_assistant).then_some(messages)
+}
+
+/// Windows Electron surface: sr-only live-region announcements delimit turns.
+/// Requires BOTH user and assistant announcements before extracting anything,
+/// so unrelated apps and older layouts keep falling through to the family and
+/// document parsers.
+fn announcement_messages(tree: &SemanticTree) -> Option<Vec<(NodeId, &'static str, String)>> {
+ let order: Vec<NodeId> = all_nodes(tree).collect();
+ let mut announcements: Vec<(usize, NodeId, &'static str, String)> = Vec::new();
+ for (position, &node) in order.iter().enumerate() {
+ let Some((actor, remainder)) = announcement(tree, node) else {
+ continue;
+ };
+ announcements.push((position, node, actor, remainder.to_owned()));
+ if announcements.len() == MAX_MESSAGES {
+ break;
+ }
+ }
+ let has_user = announcements.iter().any(|(_, _, actor, _)| *actor == "[user]");
+ let has_assistant = announcements.iter().any(|(_, _, actor, _)| *actor == "Claude");
+ if !has_user || !has_assistant {
+ return None;
+ }
+
+ let mut messages = Vec::new();
+ let mut retained_bytes = 0usize;
+ for (index, (position, node, actor, remainder)) in announcements.iter().enumerate() {
+ let window_end = announcements
+ .get(index + 1)
+ .map_or(order.len(), |next| next.0);
+ let body = if *actor == "[user]" {
+ // The announcement carries the complete user text; the visible
+ // rendering splits it across pill buttons and text fragments.
+ (!remainder.is_empty()).then(|| remainder.clone())
+ } else {
+ assistant_body(tree, &order[position + 1..window_end], &mut retained_bytes)
+ .or_else(|| (!remainder.is_empty()).then(|| remainder.clone()))
+ };
+ if let Some(body) = body {
+ messages.push((*node, *actor, body));
+ }
+ }
+ (!messages.is_empty()).then_some(messages)
+}
+
+fn announcement<'a>(tree: &'a SemanticTree, node: NodeId) -> Option<(&'static str, &'a str)> {
+ if !tree.role(node).is_some_and(|role| {
+ role.eq_ignore_ascii_case("Text") || role.eq_ignore_ascii_case("AXStaticText")
+ }) {
+ return None;
+ }
+ if !has_class(tree, node, "sr-only") {
+ return None;
+ }
+ let text = node_content(tree, node)?;
+ text.strip_prefix(USER_ANNOUNCEMENT)
+ .map(|remainder| ("[user]", remainder.trim()))
+ .or_else(|| {
+ text.strip_prefix(ASSISTANT_ANNOUNCEMENT)
+ .map(|remainder| ("Claude", remainder.trim()))
+ })
+}
+
+/// Visible assistant body between one announcement and the next: text-role
+/// lines, minus tool pills and chips (button subtrees), sr-only duplicates,
+/// window chrome, relative timestamps, and everything at or after the closing
+/// sr-only "Show message actions" button or the action-bar/composer controls.
+fn assistant_body(
+ tree: &SemanticTree,
+ window: &[NodeId],
+ retained_bytes: &mut usize,
+) -> Option<String> {
+ let mut lines = Vec::<String>::new();
+ let mut skip_under: Option<NodeId> = None;
+ for &node in window {
+ if let Some(anchor) = skip_under {
+ if is_descendant(tree, node, anchor) {
+ continue;
+ }
+ skip_under = None;
+ }
+ if tree
+ .role(node)
+ .is_some_and(|role| role.eq_ignore_ascii_case("Button"))
+ {
+ if has_class(tree, node, "sr-only") {
+ break;
+ }
+ if TURN_STOP_CLASSES
+ .iter()
+ .any(|class| has_class(tree, node, class))
+ {
+ break;
+ }
+ skip_under = Some(node);
+ continue;
+ }
+ if has_class(tree, node, "sr-only") {
+ continue;
+ }
+ if !is_message_text_role(tree.role(node)) {
+ continue;
+ }
+ let Some(content) = node_content(tree, node) else {
+ continue;
+ };
+ for line in content
+ .lines()
+ .map(str::trim)
+ .filter(|line| !line.is_empty())
+ {
+ if CHROME_LINES
+ .iter()
+ .chain(IGNORED_ACTIONS.iter())
+ .any(|chrome| line.eq_ignore_ascii_case(chrome))
+ {
+ continue;
+ }
+ if line.len() <= 24 && line.ends_with(" ago") {
+ continue;
+ }
+ if lines.last().is_some_and(|previous| previous == line) {
+ continue;
+ }
+ let separator = usize::from(!lines.is_empty());
+ let remaining = MAX_BODY_BYTES.saturating_sub(*retained_bytes + separator);
+ if remaining == 0 {
+ break;
+ }
+ let line = truncate_str(line, remaining);
+ if line.is_empty() {
+ break;
+ }
+ *retained_bytes += line.len() + separator;
+ lines.push(line.to_owned());
+ }
+ if *retained_bytes == MAX_BODY_BYTES {
+ break;
+ }
+ }
+ (!lines.is_empty()).then(|| lines.join("\n"))
+}
+
+fn is_descendant(tree: &SemanticTree, node: NodeId, ancestor: NodeId) -> bool {
+ let mut current = tree.parent(node);
+ while let Some(parent) = current {
+ if parent == ancestor {
+ return true;
+ }
+ current = tree.parent(parent);
+ }
+ false
+}
+
fn actor_for_node(tree: &SemanticTree, node: NodeId) -> Option<&'static str> {
if has_marker_class(tree, node, USER_MARKER) && has_user_bubble_ancestor(tree, node) {
Some("[user]")
@@ -272,17 +446,23 @@ fn node_content(tree: &SemanticTree, node: NodeId) -> Option<&str> {
.filter(|content| !content.is_empty())
}
-fn first_root_title(tree: &SemanticTree) -> Option<&str> {
- tree.roots().find_map(|root| {
- tree.title(root)
- .or_else(|| tree.description(root))
- .or_else(|| tree.text(root))
- .or_else(|| tree.value(root))
- .map(str::trim)
- .filter(|title| {
- !title.is_empty() && title.len() <= 240 && !title.contains(['\n', '\r'])
- })
- })
+/// `first_root_title`, but skipping roots carrying the given class. On the
+/// Windows Electron surface the roots are the three caption buttons
+/// (class `WinCaptionButton`), which would otherwise title the conversation
+/// "Minimize". Trees without that class behave exactly as before.
+fn first_root_title_excluding<'a>(tree: &'a SemanticTree, excluded_class: &str) -> Option<&'a str> {
+ tree.roots()
+ .filter(|root| !has_class(tree, *root, excluded_class))
+ .find_map(|root| {
+ tree.title(root)
+ .or_else(|| tree.description(root))
+ .or_else(|| tree.text(root))
+ .or_else(|| tree.value(root))
+ .map(str::trim)
+ .filter(|title| {
+ !title.is_empty() && title.len() <= 240 && !title.contains(['\n', '\r'])
+ })
+ })
}
fn is_message_text_role(role: Option<&str>) -> bool {
diff --git a/crates/screenpipe-semantic/src/parsers/families.rs b/crates/screenpipe-semantic/src/parsers/families.rs
index e90a9e68d..8a30f01a2 100644
--- a/crates/screenpipe-semantic/src/parsers/families.rs
+++ b/crates/screenpipe-semantic/src/parsers/families.rs
@@ -157,6 +157,13 @@ fn parse_conversation(
}
messages.push((node, actor, actor_evidence, time_label, body));
}
+ let mut title_override: Option<String> = None;
+ if messages.is_empty() {
+ if let Some(labeled) = labeled_uia_turns(profile, tree) {
+ title_override = Some(labeled.title);
+ messages = labeled.messages;
+ }
+ }
if messages.is_empty() {
return Vec::new();
}
@@ -167,11 +174,11 @@ fn parse_conversation(
format!("{}:conversation", profile.id),
IdentityQuality::Derived,
);
- conversation.title = Some(
+ conversation.title = Some(title_override.unwrap_or_else(|| {
first_root_title(tree)
.unwrap_or(profile.display_name)
- .to_owned(),
- );
+ .to_owned()
+ }));
conversation
.metadata
.insert("app".into(), profile.display_name.into());
@@ -301,7 +308,7 @@ fn parse_mail(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticI
messages.push((node, sender.to_owned(), body));
}
if messages.is_empty() {
- return Vec::new();
+ return parse_mail_list_rows(profile, tree);
}
let mut conversation = SemanticItem::new(
@@ -455,6 +462,9 @@ fn parse_calendar(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<Seman
}
fn parse_terminal(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticItem> {
+ if profile.id == "windowsterminal" {
+ return parse_windows_terminal(profile, tree);
+ }
let mut bodies = Vec::new();
for root in tree.roots() {
if let Some(body) = collect_text(tree, root) {
@@ -480,6 +490,307 @@ fn parse_terminal(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<Seman
vec![terminal]
}
+/// Windows Terminal renders each pane as a XAML `TermControl` element. Exact
+/// class match; abstain without it so WT settings/palette surfaces never fall
+/// through to a text sweep. Today the walker does not read TermControl's UIA
+/// TextPattern, so the honest yield is session identity (tab title), with the
+/// pane `Value` picked up automatically once walker-side capture lands.
+fn parse_windows_terminal(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticItem> {
+ let mut panes = Vec::new();
+ 'roots: for root in tree.roots() {
+ for node in tree.descendants(root) {
+ if tree.has_class(node, "TermControl") {
+ panes.push(node);
+ if panes.len() == MAX_STRUCTURAL_CANDIDATES {
+ break 'roots;
+ }
+ }
+ }
+ }
+ if panes.is_empty() {
+ return Vec::new();
+ }
+ let mut bodies = Vec::new();
+ for &pane in &panes {
+ let body = tree
+ .value(pane)
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ .map(str::to_owned)
+ .or_else(|| collect_text(tree, pane));
+ if let Some(body) = body {
+ bodies.push(body);
+ }
+ }
+ if bodies.is_empty() {
+ return Vec::new();
+ }
+ let title = tree
+ .text(panes[0])
+ .or_else(|| tree.description(panes[0]))
+ .map(str::trim)
+ .filter(|title| !title.is_empty() && title.len() <= 240 && !title.contains(['\n', '\r']))
+ .or_else(|| first_root_title(tree))
+ .unwrap_or(profile.display_name)
+ .to_owned();
+ let mut terminal = SemanticItem::new(
+ "terminal",
+ SemanticKind::Document,
+ format!("{}:terminal", profile.id),
+ IdentityQuality::Ephemeral,
+ );
+ terminal.title = Some(title);
+ terminal.body = Some(bodies.join("\n"));
+ terminal
+ .metadata
+ .insert("app".into(), profile.display_name.into());
+ terminal.metadata.insert("family".into(), "terminal".into());
+ terminal.source_nodes.push(panes[0]);
+ vec![terminal]
+}
+
+/// Gmail's thread list through Chromium's UIA bridge on Windows: each
+/// conversation is a `DataItem` row carrying Gmail's long-stable class markers
+/// (`zA` row, `zE` unread) whose child cells are classed `yX` (sender),
+/// `a4W` (subject + snippet), and `xW` (date). AppKit and web-DOM trees never
+/// expose the `DataItem` control type, so this path abstains there. Tokens are
+/// matched with exact `has_class`, never substring signatures — 2-3 char
+/// tokens would collide with unrelated utility classes.
+fn parse_mail_list_rows(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticItem> {
+ let mut rows = Vec::new();
+ 'roots: for root in tree.roots() {
+ for node in tree.descendants(root) {
+ if is_mail_list_row(tree, node) {
+ rows.push(node);
+ if rows.len() == MAX_STRUCTURAL_CANDIDATES {
+ break 'roots;
+ }
+ }
+ }
+ }
+ let mut entries = Vec::new();
+ for row in rows {
+ let Some(sender) = mail_row_cell_text(tree, row, "yX") else {
+ continue;
+ };
+ let Some(combined) = mail_row_cell_text(tree, row, "a4W") else {
+ continue;
+ };
+ entries.push((row, sender, combined, mail_row_cell_text(tree, row, "xW")));
+ }
+ if entries.is_empty() {
+ return Vec::new();
+ }
+ let mut mailbox = SemanticItem::new(
+ "thread-list",
+ SemanticKind::Conversation,
+ format!("{}:mail-list", profile.id),
+ IdentityQuality::Derived,
+ );
+ mailbox.title = Some(profile.display_name.to_owned());
+ mailbox
+ .metadata
+ .insert("app".into(), profile.display_name.into());
+ mailbox.metadata.insert("family".into(), "mail".into());
+ mailbox.metadata.insert("view".into(), "list".into());
+ let mut items = Vec::with_capacity(entries.len() + 1);
+ items.push(mailbox);
+ for (index, (row, sender, combined, date)) in entries.into_iter().enumerate() {
+ // Gmail joins subject and snippet with U+00A0 '-' U+00A0.
+ let (subject, snippet) = match combined.split_once("\u{a0}-\u{a0}") {
+ Some((subject, snippet)) => (subject.trim(), Some(snippet.trim())),
+ None => (combined.trim(), None),
+ };
+ if subject.is_empty() {
+ continue;
+ }
+ let mut message = SemanticItem::new(
+ format!("mail-row-{index}"),
+ SemanticKind::Message,
+ format!("{}:mail-row:{index}", profile.id),
+ IdentityQuality::Ephemeral,
+ );
+ message.parent_local_id = Some("thread-list".into());
+ message.actor = Some(sender.to_owned());
+ message.title = Some(subject.to_owned());
+ message.body = snippet.filter(|s| !s.is_empty()).map(str::to_owned);
+ if let Some(date) = date {
+ message.occurred_at = Some(date.to_owned());
+ message.timestamp_precision = Some("minute".into());
+ }
+ if tree.has_class(row, "zE") {
+ message.status = Some("unread".into());
+ }
+ message.source_nodes.push(row);
+ items.push(message);
+ }
+ if items.len() == 1 {
+ return Vec::new();
+ }
+ items
+}
+
+fn is_mail_list_row(tree: &SemanticTree, node: NodeId) -> bool {
+ tree.role(node)
+ .is_some_and(|role| role.eq_ignore_ascii_case("DataItem"))
+ && tree.has_class(node, "zA")
+ && tree.children(node).any(|cell| tree.has_class(cell, "yX"))
+ && tree.children(node).any(|cell| tree.has_class(cell, "a4W"))
+}
+
+fn mail_row_cell_text<'a>(tree: &'a SemanticTree, row: NodeId, class: &str) -> Option<&'a str> {
+ tree.children(row)
+ .find(|cell| tree.has_class(*cell, class))
+ .and_then(|cell| node_content(tree, cell))
+}
+
+struct LabeledTurns {
+ title: String,
+ messages: Vec<(NodeId, String, String)>,
+}
+
+const UIA_CONTENT_ROLES: &[&str] = &["Text", "Heading", "Link", "Paragraph", "ListItem"];
+const MAX_UIA_BUFFER_NODES: usize = 1024;
+const MAX_UIA_BODY_BYTES: usize = 48 * 1024;
+
+fn chromium_document_root(tree: &SemanticTree) -> Option<NodeId> {
+ tree.roots().find(|&root| {
+ tree.role(root)
+ .is_some_and(|role| role.eq_ignore_ascii_case("Document"))
+ && tree.identifier(root) == Some("RootWebArea")
+ })
+}
+
+fn button_label<'a>(tree: &'a SemanticTree, node: NodeId) -> Option<&'a str> {
+ tree.role(node)
+ .filter(|role| role.eq_ignore_ascii_case("Button"))?;
+ node_content(tree, node)
+}
+
+/// Chromium-UIA conversation surfaces (Codex desktop and relatives) expose no
+/// DOM classes, but their per-turn action buttons carry stable accessible
+/// names. Hard gate on the full trio before extracting; the anchor buttons
+/// then segment turns in capture order. Unrelated apps cannot satisfy the
+/// gate, and macOS trees (AXWebArea, no `RootWebArea` identifier) bypass the
+/// path entirely.
+fn labeled_uia_turns(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Option<LabeledTurns> {
+ let root = chromium_document_root(tree)?;
+ let (mut good, mut bad, mut user) = (false, false, false);
+ for node in tree.descendants(root) {
+ if let Some(label) = button_label(tree, node) {
+ good |= label.eq_ignore_ascii_case("Good response");
+ bad |= label.eq_ignore_ascii_case("Bad response");
+ user |= label.eq_ignore_ascii_case("Edit user message")
+ || label.eq_ignore_ascii_case("Copy message");
+ }
+ }
+ if !(good && bad && user) {
+ return None;
+ }
+
+ let mut messages: Vec<(NodeId, String, String)> = Vec::new();
+ let mut buffer: Vec<NodeId> = Vec::new();
+ let mut title: Option<String> = None;
+ let mut last_text: Option<NodeId> = None;
+ let mut skip_under: Option<NodeId> = None;
+ for node in tree.descendants(root) {
+ if let Some(anchor) = skip_under {
+ if is_descendant_of(tree, node, anchor) {
+ continue;
+ }
+ skip_under = None;
+ }
+ if let Some(label) = button_label(tree, node) {
+ if label.eq_ignore_ascii_case("Chat actions") {
+ // Header: the Text immediately preceding this button is the
+ // chat title.
+ title = last_text
+ .and_then(|text| node_content(tree, text))
+ .map(str::trim)
+ .filter(|text| {
+ !text.is_empty() && text.len() <= 240 && !text.contains(['\n', '\r'])
+ })
+ .map(str::to_owned);
+ buffer.clear();
+ } else if label.eq_ignore_ascii_case("Edit user message") {
+ buffer.clear();
+ if let Some(body) = collect_text(tree, node) {
+ if messages.len() < MAX_STRUCTURAL_CANDIDATES {
+ messages.push((node, "[user]".to_owned(), body));
+ }
+ }
+ skip_under = Some(node);
+ } else if [
+ "Copy message",
+ "Edit message",
+ "Bad response",
+ "Fork from this point",
+ ]
+ .iter()
+ .any(|anchor| label.eq_ignore_ascii_case(anchor))
+ {
+ // Turn-adjacent chrome (timestamps etc.) is discarded.
+ buffer.clear();
+ } else if label.eq_ignore_ascii_case("Good response") {
+ if !buffer.is_empty() && messages.len() < MAX_STRUCTURAL_CANDIDATES {
+ if let Some(body) = joined_buffer_text(tree, &buffer) {
+ messages.push((buffer[0], profile.display_name.to_owned(), body));
+ }
+ }
+ buffer.clear();
+ }
+ // Inline cards ("Scheduled task", "Worked for ...") must not
+ // split the assistant body.
+ continue;
+ }
+ let role = tree.role(node).unwrap_or("");
+ if !UIA_CONTENT_ROLES
+ .iter()
+ .any(|candidate| candidate.eq_ignore_ascii_case(role))
+ {
+ continue;
+ }
+ if node_content(tree, node).is_none() {
+ continue;
+ }
+ if role.eq_ignore_ascii_case("Text") {
+ last_text = Some(node);
+ }
+ if buffer.len() < MAX_UIA_BUFFER_NODES {
+ buffer.push(node);
+ }
+ if role.eq_ignore_ascii_case("ListItem") {
+ // Child Texts duplicate fragments of the item line.
+ skip_under = Some(node);
+ }
+ }
+ (!messages.is_empty()).then(|| LabeledTurns {
+ title: title.unwrap_or_else(|| profile.display_name.to_owned()),
+ messages,
+ })
+}
+
+fn joined_buffer_text(tree: &SemanticTree, buffer: &[NodeId]) -> Option<String> {
+ let mut lines: Vec<&str> = Vec::new();
+ let mut bytes = 0usize;
+ for &node in buffer {
+ let Some(content) = node_content(tree, node) else {
+ continue;
+ };
+ for line in content.lines().map(str::trim).filter(|line| !line.is_empty()) {
+ if lines.last() == Some(&line) {
+ continue;
+ }
+ bytes += line.len() + 1;
+ if bytes > MAX_UIA_BODY_BYTES {
+ return (!lines.is_empty()).then(|| lines.join("\n"));
+ }
+ lines.push(line);
+ }
+ }
+ (!lines.is_empty()).then(|| lines.join("\n"))
+}
+
fn leaf_marker_nodes(tree: &SemanticTree, tokens: &[&str]) -> Vec<NodeId> {
let mut candidates = Vec::new();
for root in tree.roots() {
diff --git a/crates/screenpipe-semantic/tests/builtin_catalog.rs b/crates/screenpipe-semantic/tests/builtin_catalog.rs
index fc8f00623..c9b87463e 100644
--- a/crates/screenpipe-semantic/tests/builtin_catalog.rs
+++ b/crates/screenpipe-semantic/tests/builtin_catalog.rs
@@ -22,6 +22,7 @@ const EXPECTED_TARGETS: &[&str] = &[
"claudemacapp",
"clickup",
"clickup_web",
+ "codex",
"cursor",
"discord",
"fantastical",
@@ -55,6 +56,7 @@ const EXPECTED_TARGETS: &[&str] = &[
"warp",
"whatsapp",
"whatsapp_web",
+ "windowsterminal",
"windsurf",
"xcode",
];
@@ -79,14 +81,14 @@ fn identity_for(profile: &screenpipe_semantic::parsers::BuiltinAppProfile) -> Ap
}
#[test]
-fn catalog_exactly_matches_all_47_targets() {
+fn catalog_exactly_matches_all_49_targets() {
let actual: BTreeSet<_> = builtin_app_profiles()
.iter()
.map(|profile| profile.id)
.collect();
let expected: BTreeSet<_> = EXPECTED_TARGETS.iter().copied().collect();
- assert_eq!(builtin_app_profiles().len(), 47);
- assert_eq!(actual.len(), 47, "profile ids must be unique");
+ assert_eq!(builtin_app_profiles().len(), 49);
+ assert_eq!(actual.len(), 49, "profile ids must be unique");
assert_eq!(actual, expected);
}
diff --git a/crates/screenpipe-semantic/tests/context_efficiency.rs b/crates/screenpipe-semantic/tests/context_efficiency.rs
index 53c40c10a..4c5239402 100644
--- a/crates/screenpipe-semantic/tests/context_efficiency.rs
+++ b/crates/screenpipe-semantic/tests/context_efficiency.rs
@@ -9,7 +9,7 @@ mod context_eval;
fn semantic_context_is_smaller_than_raw_without_losing_task_facts() {
let report = context_eval::evaluate_suite().expect("context eval must run");
- assert_eq!(report.catalog_profiles, 47);
+ assert_eq!(report.catalog_profiles, 49);
assert_eq!(report.parser_implementations, 23);
assert_eq!(report.representative_cases, 10);
assert_eq!(report.totals.raw_json.offscreen_distractors_retained, 10);
diff --git a/crates/screenpipe-semantic/tests/family_parsers.rs b/crates/screenpipe-semantic/tests/family_parsers.rs
index a2941b365..52e95a5d0 100644
--- a/crates/screenpipe-semantic/tests/family_parsers.rs
+++ b/crates/screenpipe-semantic/tests/family_parsers.rs
@@ -4,8 +4,8 @@
use screenpipe_semantic::{
parsers::builtin_parser_registry, AppIdentity, NodeId, OutputBudget, ParseContext,
- ParserChainResult, SemanticItem, SemanticNodeInput, SemanticTreeBuilder, TreeBudget,
- ValidatedParseOutcome,
+ ParserChainResult, SemanticItem, SemanticKind, SemanticNodeInput, SemanticTreeBuilder,
+ TreeBudget, ValidatedParseOutcome,
};
use serde::Deserialize;
@@ -159,3 +159,152 @@ fn terminal_fixture_deduplicates_adjacent_accessibility_rows() {
assert_eq!(items.len(), 1);
assert_eq!(items[0].body.as_deref(), Some("$ cargo test\n24 passed"));
}
+
+// --- Windows UIA surfaces -------------------------------------------------
+// These fixtures mirror the structure of real Windows captures (roles,
+// depths, class tokens, automation ids) with fully synthetic content.
+
+#[test]
+fn windows_terminal_fixture_extracts_session_from_termcontrol_pane() {
+ let items = handled(
+ include_str!("fixtures/families/windows_terminal_session.json"),
+ "family.terminal",
+ );
+ assert_eq!(items.len(), 1);
+ assert_eq!(items[0].kind, SemanticKind::Document);
+ assert_eq!(items[0].item_key, "windowsterminal:terminal");
+ assert_eq!(items[0].title.as_deref(), Some("Dev Shell"));
+ assert_eq!(
+ items[0].metadata.get("app").map(String::as_str),
+ Some("Windows Terminal")
+ );
+}
+
+#[test]
+fn windows_terminal_abstains_without_a_termcontrol_pane() {
+ // Same shell chrome, no terminal pane: must not sweep tab titles and
+ // caption buttons into a fake session.
+ let source = include_str!("fixtures/families/windows_terminal_session.json")
+ .replace("\"TermControl\"", "\"SettingsControl\"");
+ let result = parse_fixture(&source);
+ assert!(matches!(
+ result.outcome,
+ ValidatedParseOutcome::NotHandled | ValidatedParseOutcome::Empty
+ ));
+}
+
+#[test]
+fn gmail_windows_fixture_extracts_list_rows_with_unread_state() {
+ let items = handled(
+ include_str!("fixtures/families/gmail_inbox_windows.json"),
+ "family.mail",
+ );
+ assert!(items.len() >= 2, "expected a mailbox plus rows");
+ assert_eq!(items[0].kind, SemanticKind::Conversation);
+ assert_eq!(
+ items[0].metadata.get("view").map(String::as_str),
+ Some("list")
+ );
+ let rows = &items[1..];
+ assert!(rows.iter().all(|row| row.kind == SemanticKind::Message));
+ assert!(rows.iter().all(|row| row.actor.is_some()));
+ assert!(rows.iter().all(|row| row.title.is_some()));
+ assert!(
+ rows.iter().any(|row| row.status.as_deref() == Some("unread")),
+ "unread rows carry the zE class marker"
+ );
+ // Subject and snippet are split, never concatenated into the subject.
+ assert!(rows
+ .iter()
+ .all(|row| !row.title.as_deref().unwrap_or_default().contains('\u{a0}')));
+}
+
+#[test]
+fn chatgpt_windows_fixture_extracts_turns_from_role_description_headings() {
+ let items = handled(
+ include_str!("fixtures/families/chatgpt_windows_turns.json"),
+ "app.chatgpt.turn_markers",
+ );
+ assert_eq!(items.len(), 3);
+ assert_eq!(items[1].actor.as_deref(), Some("[user]"));
+ assert_eq!(
+ items[1].body.as_deref(),
+ Some("Plan a birthday dinner for Alex Rivera")
+ );
+ assert_eq!(items[2].actor.as_deref(), Some("ChatGPT"));
+ let assistant = items[2].body.as_deref().unwrap_or_default();
+ assert!(assistant.contains("three restaurant ideas"));
+ // Anchor text, not the href; sidebar/composer chrome excluded by class.
+ assert!(assistant.contains("Sample menu"));
+ assert!(!assistant.contains("https://"));
+ for item in &items {
+ let body = item.body.as_deref().unwrap_or_default();
+ for chrome in [
+ "Get responses tailored to you",
+ "Help center",
+ "Ask anything",
+ "New Chrome available",
+ ] {
+ assert!(!body.contains(chrome), "leaked browser/sidebar chrome: {chrome}");
+ }
+ }
+}
+
+#[test]
+fn chatgpt_windows_landing_surface_is_empty_not_abstained() {
+ let result = parse_fixture(include_str!("fixtures/families/chatgpt_windows_landing.json"));
+ assert_eq!(
+ result.selected_parser_id.as_deref(),
+ Some("app.chatgpt.turn_markers")
+ );
+ assert!(matches!(result.outcome, ValidatedParseOutcome::Empty));
+}
+
+#[test]
+fn codex_windows_fixture_extracts_labeled_uia_turns() {
+ let items = handled(
+ include_str!("fixtures/families/codex_windows_conversation.json"),
+ "family.conversation",
+ );
+ assert!(items.len() >= 2, "conversation plus the assistant turn");
+ assert_eq!(items[0].kind, SemanticKind::Conversation);
+ assert_eq!(items[0].title.as_deref(), Some("Plan weekend trip"));
+ // The Windows walker retains no text under the user bubble on this
+ // surface — only the "Edit user message" control itself — so no user turn
+ // is emitted. Verified against real Codex captures; the anchor is wired
+ // so a user turn appears automatically if that text ever survives.
+ assert!(items
+ .iter()
+ .all(|item| item.actor.as_deref() != Some("[user]")));
+ let assistant = items
+ .iter()
+ .find(|item| item.actor.as_deref() == Some("Codex"))
+ .and_then(|item| item.body.as_deref())
+ .unwrap_or_default();
+ assert!(assistant.contains("two-day plan"));
+ assert!(assistant.contains("Morning: leave by 8 AM"));
+ for chrome in [
+ "Ask for follow-up changes",
+ "acme workspace",
+ "Refactor billing scripts",
+ "10:57 AM",
+ ] {
+ assert!(
+ !assistant.contains(chrome),
+ "leaked sidebar/composer chrome: {chrome}"
+ );
+ }
+}
+
+#[test]
+fn codex_windows_abstains_without_the_response_button_trio() {
+ // Remove the rating anchors: the tree is still a Chromium document with
+ // plenty of text, and must not produce a conversation.
+ let source = include_str!("fixtures/families/codex_windows_conversation.json")
+ .replace("\"Good response\"", "\"Some button\"");
+ let result = parse_fixture(&source);
+ assert!(matches!(
+ result.outcome,
+ ValidatedParseOutcome::NotHandled | ValidatedParseOutcome::Empty
+ ));
+}
diff --git a/crates/screenpipe-semantic/tests/fixtures/apps/claude_windows_announcements.json b/crates/screenpipe-semantic/tests/fixtures/apps/claude_windows_announcements.json
new file mode 100644
index 000000000..ab17ca9c9
--- /dev/null
+++ b/crates/screenpipe-semantic/tests/fixtures/apps/claude_windows_announcements.json
@@ -0,0 +1,187 @@
+{
+ "app": {
+ "platform": "windows",
+ "app_id": null,
+ "executable": "Claude.exe",
+ "display_name": "Claude",
+ "version": "1.0.0",
+ "browser_url": "https://claude.ai/epitaxy/local_00000000-0000-0000-0000-000000000000"
+ },
+ "nodes": [
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Minimize",
+ "classes": [
+ "WinCaptionButton"
+ ]
+ },
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Restore",
+ "classes": [
+ "WinCaptionButton"
+ ]
+ },
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Close",
+ "classes": [
+ "WinCaptionButton"
+ ]
+ },
+ {
+ "parent": 2,
+ "role": "Document",
+ "identifier": "RootWebArea",
+ "text": "file:///app/resources/app.asar/.vite/renderer/main_window/index.html"
+ },
+ {
+ "parent": 3,
+ "role": "Button",
+ "text": "New",
+ "classes": [
+ "w-full",
+ "shrink-0",
+ "border-none",
+ "text-left"
+ ]
+ },
+ {
+ "parent": 3,
+ "role": "Button",
+ "text": "Session actions, new activity",
+ "identifier": "base-ui-_r_1_",
+ "classes": [
+ "cds-reset",
+ "group/btn",
+ "relative",
+ "isolate",
+ "inline-flex"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Button",
+ "text": "Ran a command, used a tool",
+ "classes": [
+ "relative",
+ "group/tool",
+ "flex",
+ "self-start"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Button",
+ "text": "Show message actions",
+ "classes": [
+ "sr-only"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Text",
+ "text": "You said: /plan draft the beach cleanup email",
+ "classes": [
+ "sr-only",
+ "select-none"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Button",
+ "text": "plan",
+ "identifier": "base-ui-_r_2_",
+ "classes": [
+ "group",
+ "inline-flex",
+ "relative",
+ "text-accent-100"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Text",
+ "text": "draft the beach cleanup email"
+ },
+ {
+ "parent": 5,
+ "role": "Button",
+ "text": "Show message actions",
+ "classes": [
+ "sr-only"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Text",
+ "text": "Claude responded: Drafted the note for Alex Rivera and",
+ "classes": [
+ "sr-only",
+ "select-none"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Button",
+ "text": "Wrote the draft",
+ "classes": [
+ "relative",
+ "group/tool",
+ "flex",
+ "self-start"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Text",
+ "text": "Drafted the note for Alex Rivera and set the reply address to [email protected]. Ready to send tomorrow."
+ },
+ {
+ "parent": 14,
+ "role": "Text",
+ "text": "cleanup-draft.md"
+ },
+ {
+ "parent": 14,
+ "role": "Button",
+ "text": "Copy message",
+ "classes": [
+ "group/btn",
+ "relative",
+ "isolate"
+ ]
+ },
+ {
+ "parent": 14,
+ "role": "Text",
+ "text": "7 hours ago"
+ },
+ {
+ "parent": 5,
+ "role": "Button",
+ "text": "Show message actions",
+ "classes": [
+ "sr-only"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Text",
+ "text": "Chat mode"
+ },
+ {
+ "parent": 5,
+ "role": "Button",
+ "text": "Send",
+ "classes": [
+ "group/btn",
+ "relative",
+ "isolate"
+ ]
+ }
+ ]
+}
diff --git a/crates/screenpipe-semantic/tests/fixtures/families/chatgpt_windows_landing.json b/crates/screenpipe-semantic/tests/fixtures/families/chatgpt_windows_landing.json
new file mode 100644
index 000000000..e56d432e7
--- /dev/null
+++ b/crates/screenpipe-semantic/tests/fixtures/families/chatgpt_windows_landing.json
@@ -0,0 +1,20 @@
+{
+ "app": {
+ "platform": "windows",
+ "app_id": null,
+ "executable": "chrome.exe",
+ "display_name": "Google Chrome",
+ "version": null,
+ "browser_url": "https://chatgpt.com/"
+ },
+ "nodes": [
+ { "parent": null, "role": "Button", "text": "Minimize", "identifier": "view_1", "classes": ["WindowsCaptionButton"], "description": "button" },
+ { "parent": null, "role": "Button", "text": "Close", "identifier": "view_4", "classes": ["WindowsCaptionButton"], "description": "button" },
+ { "parent": 1, "role": "Edit", "text": "chatgpt.com", "value": "chatgpt.com", "identifier": "view_1012", "classes": ["OmniboxViewViews"], "description": "Ask Google or type a URL" },
+ { "parent": 1, "role": "Button", "text": "New Chrome available", "identifier": "view_1007", "classes": ["BrowserAppMenuButton"], "description": "button" },
+ { "parent": 3, "role": "Text", "text": "Where should we begin?", "classes": ["wm-app-landingHeading"], "description": "heading" },
+ { "parent": 3, "role": "Text", "text": "Get responses tailored to you", "identifier": "sidebar-login-title", "classes": ["wm-sidebar-loginTitle"], "description": "heading" },
+ { "parent": 3, "role": "Hyperlink", "text": "Help center", "value": "https://help.example.com/", "classes": ["wm-sidebar-navigationLink"], "description": "link" },
+ { "parent": 3, "role": "Edit", "text": "Ask anything", "identifier": "mobile-composer-prompt", "classes": ["wm-composer-textarea"], "description": "Ask anything" }
+ ]
+}
diff --git a/crates/screenpipe-semantic/tests/fixtures/families/chatgpt_windows_turns.json b/crates/screenpipe-semantic/tests/fixtures/families/chatgpt_windows_turns.json
new file mode 100644
index 000000000..724735bf8
--- /dev/null
+++ b/crates/screenpipe-semantic/tests/fixtures/families/chatgpt_windows_turns.json
@@ -0,0 +1,130 @@
+{
+ "app": {
+ "platform": "windows",
+ "app_id": null,
+ "executable": "chrome.exe",
+ "display_name": "Google Chrome",
+ "version": null,
+ "browser_url": "https://chatgpt.com/c/synthetic-fixture"
+ },
+ "nodes": [
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Minimize",
+ "identifier": "view_1",
+ "classes": [
+ "WindowsCaptionButton"
+ ],
+ "description": "button"
+ },
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Close",
+ "identifier": "view_4",
+ "classes": [
+ "WindowsCaptionButton"
+ ],
+ "description": "button"
+ },
+ {
+ "parent": 1,
+ "role": "Button",
+ "text": "Back",
+ "identifier": "view_1001",
+ "classes": [
+ "BackForwardButton"
+ ],
+ "description": "button"
+ },
+ {
+ "parent": 2,
+ "role": "Edit",
+ "text": "chatgpt.com",
+ "value": "chatgpt.com",
+ "identifier": "view_1012",
+ "classes": [
+ "OmniboxViewViews"
+ ],
+ "description": "Ask Google or type a URL"
+ },
+ {
+ "parent": 1,
+ "role": "Button",
+ "text": "New Chrome available",
+ "identifier": "view_1007",
+ "classes": [
+ "BrowserAppMenuButton"
+ ],
+ "description": "button"
+ },
+ {
+ "parent": 4,
+ "role": "Text",
+ "text": "You said:",
+ "description": "heading"
+ },
+ {
+ "parent": 5,
+ "role": "Text",
+ "text": "Plan a birthday dinner for Alex Rivera",
+ "description": "text"
+ },
+ {
+ "parent": 4,
+ "role": "Text",
+ "text": "ChatGPT said:",
+ "description": "heading"
+ },
+ {
+ "parent": 7,
+ "role": "Text",
+ "text": "Here are three restaurant ideas near the waterfront.",
+ "description": "text"
+ },
+ {
+ "parent": 7,
+ "role": "Hyperlink",
+ "text": "Sample menu",
+ "value": "https://example.com/menu",
+ "description": "link"
+ },
+ {
+ "parent": 7,
+ "role": "Button",
+ "text": "Copy",
+ "description": "button"
+ },
+ {
+ "parent": 7,
+ "role": "Text",
+ "text": "Get responses tailored to you",
+ "identifier": "sidebar-login-title",
+ "classes": [
+ "wm-sidebar-loginTitle"
+ ],
+ "description": "heading"
+ },
+ {
+ "parent": 7,
+ "role": "Hyperlink",
+ "text": "Help center",
+ "value": "https://help.example.com/",
+ "classes": [
+ "wm-sidebar-navigationLink"
+ ],
+ "description": "link"
+ },
+ {
+ "parent": 7,
+ "role": "Edit",
+ "text": "Ask anything",
+ "identifier": "mobile-composer-prompt",
+ "classes": [
+ "wm-composer-textarea"
+ ],
+ "description": "Ask anything"
+ }
+ ]
+}
diff --git a/crates/screenpipe-semantic/tests/fixtures/families/codex_windows_conversation.json b/crates/screenpipe-semantic/tests/fixtures/families/codex_windows_conversation.json
new file mode 100644
index 000000000..a2d4a0a6a
--- /dev/null
+++ b/crates/screenpipe-semantic/tests/fixtures/families/codex_windows_conversation.json
@@ -0,0 +1,359 @@
+{
+ "app": {
+ "platform": "windows",
+ "app_id": null,
+ "executable": "C:\\Program Files\\Codex\\Codex.exe",
+ "display_name": "Codex",
+ "version": "0.4.12",
+ "browser_url": null
+ },
+ "nodes": [
+ {
+ "parent": null,
+ "role": "Document",
+ "identifier": "RootWebArea",
+ "text": "app://-/index.html",
+ "value": "app://-/index.html",
+ "description": "document"
+ },
+ {
+ "parent": 0,
+ "role": "Button",
+ "text": "Hide sidebar",
+ "classes": [
+ "border-token-border",
+ "no-drag",
+ "cursor-interaction"
+ ]
+ },
+ {
+ "parent": 0,
+ "role": "Button",
+ "text": "New chat",
+ "classes": [
+ "focus-visible:outline-token-border",
+ "relative"
+ ]
+ },
+ {
+ "parent": 0,
+ "role": "Button",
+ "text": "Pinned",
+ "classes": [
+ "group/section-toggle",
+ "flex",
+ "min-w-0"
+ ]
+ },
+ {
+ "parent": 3,
+ "role": "ListItem",
+ "text": "Refactor billing scripts1mo",
+ "classes": [
+ "after:block",
+ "after:h-px"
+ ]
+ },
+ {
+ "parent": 4,
+ "role": "Button",
+ "text": "Unpin chat Archive chat 1mo",
+ "classes": [
+ "cursor-grab"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Button",
+ "text": "Unpin chat",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "Button",
+ "text": "Archive chat",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 3,
+ "role": "ListItem",
+ "text": "Draft trip checklist2w",
+ "classes": [
+ "after:block",
+ "after:h-px"
+ ]
+ },
+ {
+ "parent": 8,
+ "role": "Button",
+ "text": "Pin chat Archive chat 2w",
+ "classes": [
+ "group",
+ "relative"
+ ]
+ },
+ {
+ "parent": 9,
+ "role": "Button",
+ "text": "Pin chat",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 9,
+ "role": "Button",
+ "text": "Archive chat",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 0,
+ "role": "Button",
+ "text": "Open settings",
+ "identifier": "radix-_r_54_",
+ "classes": [
+ "outline-hidden",
+ "cursor-interaction"
+ ]
+ },
+ {
+ "parent": 12,
+ "role": "Text",
+ "text": "A"
+ },
+ {
+ "parent": 12,
+ "role": "Text",
+ "text": "acme workspace"
+ },
+ {
+ "parent": 0,
+ "role": "Button",
+ "text": "Update",
+ "classes": [
+ "no-drag",
+ "ease-basic",
+ "relative"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Text",
+ "text": "Plan weekend trip"
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Chat actions",
+ "identifier": "radix-_r_4e0_",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Toggle pinned summary",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Edit user message",
+ "classes": [
+ "bg-token-foreground/5",
+ "max-w-[77%]",
+ "min-w-0",
+ "overflow-hidden",
+ "break-words",
+ "rounded-2xl"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Text",
+ "text": "10:57 AM"
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Copy message",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Edit message",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Worked for 1m 12s",
+ "classes": [
+ "text-size-chat",
+ "inline-flex"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Text",
+ "text": "Here is a simple two-day plan for the coast trip."
+ },
+ {
+ "parent": 15,
+ "role": "Text",
+ "text": "Day one focuses on the drive and a short hike."
+ },
+ {
+ "parent": 15,
+ "role": "ListItem",
+ "text": "Morning: leave by 8 AM and stop at the overlook.",
+ "classes": [
+ "_markdownText_ab1cd_171",
+ "_listItem_ab1cd_335"
+ ]
+ },
+ {
+ "parent": 26,
+ "role": "Text",
+ "text": "Morning"
+ },
+ {
+ "parent": 15,
+ "role": "ListItem",
+ "text": "Afternoon: check in, then walk the harbor loop.",
+ "classes": [
+ "_markdownText_ab1cd_171",
+ "_listItem_ab1cd_335"
+ ]
+ },
+ {
+ "parent": 28,
+ "role": "Text",
+ "text": "Afternoon"
+ },
+ {
+ "parent": 15,
+ "role": "Text",
+ "text": "Pack light layers because evenings get cold."
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Scheduled task",
+ "classes": [
+ "w-full",
+ "cursor-interaction",
+ "text-left"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Copy",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Good response",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Bad response",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Fork from this point",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Text",
+ "text": "10:59 AM"
+ },
+ {
+ "parent": 15,
+ "role": "Text",
+ "text": "Ask for follow-up changes"
+ },
+ {
+ "parent": 15,
+ "role": "Button",
+ "text": "Dictate",
+ "classes": [
+ "border-token-border",
+ "no-drag"
+ ]
+ },
+ {
+ "parent": 15,
+ "role": "Text",
+ "text": "No artifacts yet"
+ },
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Minimize",
+ "identifier": "view_1",
+ "classes": [
+ "ChromeNodeCaptionButton"
+ ]
+ },
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Restore",
+ "identifier": "view_3",
+ "classes": [
+ "ChromeNodeCaptionButton"
+ ]
+ },
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Close",
+ "identifier": "view_4",
+ "classes": [
+ "ChromeNodeCaptionButton"
+ ]
+ }
+ ]
+}
diff --git a/crates/screenpipe-semantic/tests/fixtures/families/gmail_inbox_windows.json b/crates/screenpipe-semantic/tests/fixtures/families/gmail_inbox_windows.json
new file mode 100644
index 000000000..d93df29ef
--- /dev/null
+++ b/crates/screenpipe-semantic/tests/fixtures/families/gmail_inbox_windows.json
@@ -0,0 +1,255 @@
+{
+ "app": {
+ "platform": "windows",
+ "app_id": null,
+ "executable": "chrome.exe",
+ "display_name": "Chrome",
+ "version": null,
+ "browser_url": "https://mail.google.com/mail/u/0/#inbox"
+ },
+ "nodes": [
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Minimize",
+ "description": "button",
+ "identifier": "view_1",
+ "classes": [
+ "WindowsCaptionButton"
+ ]
+ },
+ {
+ "parent": 0,
+ "role": "TabItem",
+ "text": "Inbox (2) - [email protected] - Gmail",
+ "description": "tab",
+ "identifier": "view_20",
+ "classes": [
+ "Tab"
+ ]
+ },
+ {
+ "parent": 1,
+ "role": "Edit",
+ "text": "mail.google.com/mail/u/0/#inbox",
+ "description": "edit",
+ "classes": [
+ "OmniboxViewViews"
+ ]
+ },
+ {
+ "parent": 2,
+ "role": "Text",
+ "text": "Primary",
+ "description": "heading",
+ "classes": [
+ "aRz",
+ "J-KU"
+ ]
+ },
+ {
+ "parent": 3,
+ "role": "DataItem",
+ "text": "unread, Alex Rivera , Quarterly metrics review , 9:14 AM , Hi team, the Q3 dashboard numbers are ready for your review.",
+ "description": "row",
+ "identifier": ":r1",
+ "classes": [
+ "zA",
+ "zE",
+ "s00Hgd"
+ ]
+ },
+ {
+ "parent": 4,
+ "role": "DataItem",
+ "text": "unread, Alex Rivera , Quarterly metrics review , 9:14 AM , Hi team, the Q3 dashboard numbers are ready for your review.",
+ "description": "item",
+ "identifier": ":r1c",
+ "classes": [
+ "oZ-x3",
+ "xY"
+ ]
+ },
+ {
+ "parent": 5,
+ "role": "CheckBox",
+ "text": "unread, Alex Rivera , Quarterly metrics review , 9:14 AM , Hi team, the Q3 dashboard numbers are ready for your review.",
+ "description": "check box",
+ "identifier": ":r1d",
+ "classes": [
+ "oZ-jc",
+ "T-Jo",
+ "J-J5-Ji"
+ ]
+ },
+ {
+ "parent": 4,
+ "role": "DataItem",
+ "text": "Not starred",
+ "description": "item",
+ "classes": [
+ "apU",
+ "xY"
+ ]
+ },
+ {
+ "parent": 7,
+ "role": "Button",
+ "text": "Not starred",
+ "description": "button",
+ "identifier": ":r1e",
+ "classes": [
+ "aXw",
+ "T-KT"
+ ]
+ },
+ {
+ "parent": 4,
+ "role": "DataItem",
+ "text": "Not important",
+ "description": "item",
+ "classes": [
+ "WA",
+ "xY"
+ ]
+ },
+ {
+ "parent": 4,
+ "role": "DataItem",
+ "text": "Alex Rivera",
+ "description": "item",
+ "classes": [
+ "yX",
+ "xY",
+ "ulKHrd"
+ ]
+ },
+ {
+ "parent": 4,
+ "role": "DataItem",
+ "text": "Quarterly metrics review  -  Hi team, the Q3 dashboard numbers are ready for your review.",
+ "description": "item",
+ "identifier": ":r1s",
+ "classes": [
+ "xY",
+ "a4W"
+ ]
+ },
+ {
+ "parent": 11,
+ "role": "Hyperlink",
+ "text": "Quarterly metrics review  -  Hi team, the Q3 dashboard numbers are ready for your review.",
+ "description": "link",
+ "classes": [
+ "xS"
+ ]
+ },
+ {
+ "parent": 4,
+ "role": "DataItem",
+ "text": "Mon, Jul 27, 2026, 9:14 AM",
+ "description": "item",
+ "classes": [
+ "xW",
+ "xY"
+ ]
+ },
+ {
+ "parent": 3,
+ "role": "DataItem",
+ "text": "unread, Priya Shah , Lunch on Thursday? , 8:02 AM , Want to try the new ramen place near the office?",
+ "description": "row",
+ "identifier": ":r2",
+ "classes": [
+ "zA",
+ "zE",
+ "s00Hgd"
+ ]
+ },
+ {
+ "parent": 14,
+ "role": "DataItem",
+ "text": "Priya Shah",
+ "description": "item",
+ "classes": [
+ "yX",
+ "xY"
+ ]
+ },
+ {
+ "parent": 14,
+ "role": "DataItem",
+ "text": "Lunch on Thursday?  -  Want to try the new ramen place near the office?",
+ "description": "item",
+ "identifier": ":r2s",
+ "classes": [
+ "xY",
+ "a4W"
+ ]
+ },
+ {
+ "parent": 14,
+ "role": "DataItem",
+ "text": "Mon, Jul 27, 2026, 8:02 AM",
+ "description": "item",
+ "classes": [
+ "xW",
+ "xY"
+ ]
+ },
+ {
+ "parent": 3,
+ "role": "DataItem",
+ "text": "Sam Okafor , Invoice 0042 paid , Jul 26 , Payment received, receipt attached for your records.",
+ "description": "row",
+ "identifier": ":r3",
+ "classes": [
+ "zA",
+ "yO"
+ ]
+ },
+ {
+ "parent": 18,
+ "role": "DataItem",
+ "text": "Sam Okafor",
+ "description": "item",
+ "classes": [
+ "yX",
+ "xY"
+ ]
+ },
+ {
+ "parent": 18,
+ "role": "DataItem",
+ "text": "Invoice 0042 paid  -  Payment received, receipt attached for your records.",
+ "description": "item",
+ "identifier": ":r3s",
+ "classes": [
+ "xY",
+ "a4W"
+ ]
+ },
+ {
+ "parent": 18,
+ "role": "DataItem",
+ "text": "Sun, Jul 26, 2026, 4:40 PM",
+ "description": "item",
+ "classes": [
+ "xW",
+ "xY"
+ ]
+ },
+ {
+ "parent": 3,
+ "role": "Button",
+ "text": "Show more messages",
+ "description": "button",
+ "identifier": ":ls",
+ "classes": [
+ "J-J5-Ji",
+ "amH",
+ "J-JN-I"
+ ]
+ }
+ ]
+}
diff --git a/crates/screenpipe-semantic/tests/fixtures/families/windows_terminal_session.json b/crates/screenpipe-semantic/tests/fixtures/families/windows_terminal_session.json
new file mode 100644
index 000000000..8fbfaf1b1
--- /dev/null
+++ b/crates/screenpipe-semantic/tests/fixtures/families/windows_terminal_session.json
@@ -0,0 +1,74 @@
+{
+ "app": {
+ "platform": "windows",
+ "app_id": null,
+ "executable": "WindowsTerminal.exe",
+ "display_name": "Windows Terminal",
+ "version": null,
+ "browser_url": null
+ },
+ "nodes": [
+ {
+ "parent": null,
+ "role": "TabItem",
+ "text": "Dev Shell",
+ "description": "tab item",
+ "classes": [
+ "ListViewItem"
+ ]
+ },
+ {
+ "parent": 0,
+ "role": "Text",
+ "text": "Dev Shell",
+ "identifier": "HeaderTextBlock",
+ "description": "text",
+ "classes": [
+ "TextBlock"
+ ]
+ },
+ {
+ "parent": 0,
+ "role": "Button",
+ "text": "Close Tab",
+ "identifier": "CloseButton",
+ "description": "button",
+ "classes": [
+ "Button"
+ ]
+ },
+ {
+ "parent": null,
+ "role": "Text",
+ "text": "Dev Shell",
+ "description": "Dev Shell",
+ "classes": [
+ "TermControl"
+ ]
+ },
+ {
+ "parent": null,
+ "role": "MenuItem",
+ "text": "System",
+ "description": "menu item"
+ },
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Minimize",
+ "description": "button"
+ },
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Maximize",
+ "description": "button"
+ },
+ {
+ "parent": null,
+ "role": "Button",
+ "text": "Close",
+ "description": "button"
+ }
+ ]
+}
diff --git a/docs/SEMANTIC_APP_PARSER_SPEC.md b/docs/SEMANTIC_APP_PARSER_SPEC.md
index 696b48f0d..a27867807 100644
--- a/docs/SEMANTIC_APP_PARSER_SPEC.md
+++ b/docs/SEMANTIC_APP_PARSER_SPEC.md
@@ -96,19 +96,19 @@ of app definitions.
### Built-in catalog and reference family parsers
-The built-in catalog covers 47 app targets using public app identities, URL
+The built-in catalog covers 49 app targets using public app identities, URL
patterns, and stable accessibility contracts. Implementations remain
Screenpipe-owned parser families and exact app overrides.
| Family | Built-in profiles |
|---|---|
-| Conversation | Antigravity, Antigravity IDE, ChatGPT, ChatGPT legacy, ChatGPT web, Claude, Claude macOS, ClickUp, ClickUp web, Cursor, Discord, Gemini desktop, Gemini web, Messages, Messenger, Microsoft Teams, Slack, WhatsApp, WhatsApp web, Windsurf |
+| Conversation | Antigravity, Antigravity IDE, ChatGPT, ChatGPT legacy, ChatGPT web, Claude, Claude macOS, ClickUp, ClickUp web, Codex, Cursor, Discord, Gemini desktop, Gemini web, Messages, Messenger, Microsoft Teams, Slack, WhatsApp, WhatsApp web, Windsurf |
| Mail | Gmail, Mail, Microsoft Outlook, Spark Desktop, Spark Mail Classic, Superhuman |
| Editor | Antigravity IDE, Cursor, VS Code, Windsurf, Xcode |
| Document | Antigravity IDE, Claude macOS, ClickUp, ClickUp web, Microsoft Outlook, Microsoft Word, Microsoft Word web, Notes, Notion, Obsidian, Pages, TextEdit, Xcode |
| Task | Antigravity, Asana, Asana web, ClickUp, ClickUp web, Microsoft To Do, OmniFocus, Todoist, Toggl |
| Calendar | Calendar, Fantastical |
-| Terminal | Ghostty, iTerm2, Terminal, Warp |
+| Terminal | Ghostty, iTerm2, Terminal, Warp, Windows Terminal |
Profiles may belong to more than one family because the same app can expose
different semantic surfaces. The registry still runs at most four matching