YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(semantic): repair priority parser coverage
Louis Beaumont3075996602cf
diff --git a/crates/screenpipe-engine/src/routes/request_origin.rs b/crates/screenpipe-engine/src/routes/request_origin.rsindex 69d79d0db..0303c91ec 100644--- a/crates/screenpipe-engine/src/routes/request_origin.rs+++ b/crates/screenpipe-engine/src/routes/request_origin.rs@@ -6,16 +6,32 @@ use axum::{extract::FromRequestParts, http::request::Parts};const CLIENT_HEADER: &str = "x-screenpipe-client";-/// Explicit attribution for user-initiated direct API retrievals.+/// Privacy-safe attribution for retrieval requests.///-/// Missing and unknown values stay unattributed. This prevents app requests,-/// background Pipes, health checks, and arbitrary user-agent strings from-/// entering the Repeat Value numerator as direct API use.-pub struct ExplicitApiClient(bool);+/// Only a tiny fixed enum is accepted from the client header. Missing and+/// unknown values are grouped as app-originated requests so arbitrary+/// user-agent strings never become analytics dimensions, and only the fixed+/// `api` value enters the Repeat Value direct API numerator.+pub struct ExplicitApiClient(RequestSource);++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+enum RequestSource {+ App,+ Mcp,+ Api,+}impl ExplicitApiClient {pub(crate) fn is_direct_api(&self) -> bool {- self.0+ self.0 == RequestSource::Api+ }++ pub(crate) fn source_label(&self) -> &'static str {+ match self.0 {+ RequestSource::App => "app",+ RequestSource::Mcp => "MCP",+ RequestSource::Api => "API",+ }}}@@ -33,12 +49,21 @@ impl<S: Send + Sync> FromRequestParts<S> for ExplicitApiClient {'b: 'c,{Box::pin(async move {- let is_direct_api = parts+ let source = parts.headers.get(CLIENT_HEADER).and_then(|value| value.to_str().ok())- .is_some_and(|value| value == "api");- Ok(Self(is_direct_api))+ .map(|value| {+ if value.eq_ignore_ascii_case("api") {+ RequestSource::Api+ } else if value.eq_ignore_ascii_case("mcp") {+ RequestSource::Mcp+ } else {+ RequestSource::App+ }+ })+ .unwrap_or(RequestSource::App);+ Ok(Self(source))})}}@@ -65,9 +90,16 @@ mod tests {#[tokio::test]async fn accepts_only_the_fixed_direct_api_enum() {assert!(extract(Some("api")).await.is_direct_api());+ assert_eq!(extract(Some("api")).await.source_label(), "API");+ assert_eq!(extract(Some("mcp")).await.source_label(), "MCP");assert!(!extract(Some("claude-private-project")).await.is_direct_api());+ assert_eq!(+ extract(Some("claude-private-project")).await.source_label(),+ "app"+ );assert!(!extract(None).await.is_direct_api());+ assert_eq!(extract(None).await.source_label(), "app");}}diff --git a/crates/screenpipe-engine/src/routes/search.rs b/crates/screenpipe-engine/src/routes/search.rsindex e3d01cbf7..b57ceb61f 100644--- a/crates/screenpipe-engine/src/routes/search.rs+++ b/crates/screenpipe-engine/src/routes/search.rs@@ -557,6 +557,26 @@ fn capture_direct_api_search_value(client: &ExplicitApiClient, result_count: usi}}+fn parsed_parser_profiles(content_items: &[ContentItem]) -> Vec<String> {+ let mut parser_ids = Vec::new();+ for item in content_items {+ let ContentItem::Parsed(content) = item else {+ continue;+ };+ if parser_ids+ .iter()+ .any(|parser_id| parser_id == &content.parser_id)+ {+ continue;+ }+ parser_ids.push(content.parser_id.clone());+ if parser_ids.len() == 16 {+ break;+ }+ }+ parser_ids+}+/// Middle-truncate a string to at most `max_chars` characters./// Keeps the first half and last half, inserting a marker in between./// Safe on UTF-8 char boundaries.@@ -1373,15 +1393,26 @@ pub(crate) async fn search(debug!("search completed: found {} results", total);+ let response_non_empty = !content_items.is_empty();+ let parsed_parser_profiles = if parsed_search {+ parsed_parser_profiles(&content_items)+ } else {+ Vec::new()+ };+// Track search analyticsanalytics::capture_event_nonblocking("search_performed",serde_json::json!({"query_length": query.q.as_ref().map(|q| q.len()).unwrap_or(0),"content_type": format!("{:?}", query.content_type),+ "request_source": api_client.source_label(),"has_date_filter": query.start_time.is_some() || query.end_time.is_some(),"has_app_filter": query.app_name.is_some(),"result_count": total,+ "non_empty": response_non_empty,+ "parsed_non_empty": parsed_search && response_non_empty,+ "parsed_parser_profiles": parsed_parser_profiles,"limit": query.pagination.limit,"offset": query.pagination.offset,}),diff --git a/crates/screenpipe-semantic/src/parsers/families.rs b/crates/screenpipe-semantic/src/parsers/families.rsindex 4a4653590..a22e7c17c 100644--- a/crates/screenpipe-semantic/src/parsers/families.rs+++ b/crates/screenpipe-semantic/src/parsers/families.rs@@ -169,6 +169,9 @@ fn parse_conversation(messages = fluent_control_message_turns(tree);}if messages.is_empty() {+ if let Some(chat_list) = fluent_chat_list(profile, tree) {+ return chat_list;+ }return chromium_chat_list(profile, tree).unwrap_or_default();}@@ -562,12 +565,12 @@ fn parse_windows_terminal(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vvec![terminal]}-/// Gmail's thread list through Chromium's UIA bridge on Windows: each-/// conversation is a `DataItem` row carrying Gmail's long-stable class markers-/// (`zA` row, `zE` unread) whose child cells are classed `yX` (sender),-/// `a4W` (subject + snippet), and `xW` (date). AppKit and web-DOM trees never-/// expose the `DataItem` control type, so this path abstains there. Tokens are-/// matched with exact `has_class`, never substring signatures — 2-3 char+/// Gmail's thread list through Chromium's UIA bridge: each conversation row+/// carries Gmail's long-stable class markers (`zA` row, `zE` unread) whose+/// child cells are classed `yX` (sender), `a4W` (subject + snippet), and `xW`+/// (date). Windows publishes these rows as `DataItem`; macOS/AX and browser+/// bridges can expose the same DOM row as a generic group/row/list item. Tokens+/// are matched with exact `has_class`, never substring signatures -- 2-3 char/// tokens would collide with unrelated utility classes.fn parse_mail_list_rows(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticItem> {let mut rows = Vec::new();@@ -644,13 +647,18 @@ fn parse_mail_list_rows(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec}fn is_mail_list_row(tree: &SemanticTree, node: NodeId) -> bool {- tree.role(node)- .is_some_and(|role| role.eq_ignore_ascii_case("DataItem"))+ tree.role(node).is_some_and(is_gmail_row_role)&& tree.has_class(node, "zA")&& tree.children(node).any(|cell| tree.has_class(cell, "yX"))&& tree.children(node).any(|cell| tree.has_class(cell, "a4W"))}+fn is_gmail_row_role(role: &str) -> bool {+ ["DataItem", "AXGroup", "AXRow", "Group", "ListItem", "Row"]+ .iter()+ .any(|candidate| role.eq_ignore_ascii_case(candidate))+}+fn mail_row_cell_text<'a>(tree: &'a SemanticTree, row: NodeId, class: &str) -> Option<&'a str> {tree.children(row).find(|cell| tree.has_class(*cell, class))@@ -879,6 +887,186 @@ fn fluent_message_sender<'a>(})}+struct FluentChatListEntry {+ node: NodeId,+ title: String,+ last_sender: Option<String>,+ last_message: Option<String>,+ last_active: Option<String>,+ status: Option<String>,+ chat_type: &'static str,+}++fn fluent_chat_list(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Option<Vec<SemanticItem>> {+ if profile.id != "microsoftteams" || tree_any(tree, |node| is_fluent_message_anchor(tree, node))+ {+ return None;+ }++ let mut has_new_message_control = false;+ let mut entries = Vec::new();+ 'roots: for root in tree.roots() {+ for node in tree.descendants(root) {+ if button_label(tree, node).is_some_and(|label| {+ label+ .trim_start()+ .get(..11)+ .is_some_and(|prefix| prefix.eq_ignore_ascii_case("new message"))+ }) {+ has_new_message_control = true;+ continue;+ }+ if !tree+ .role(node)+ .is_some_and(|role| role.eq_ignore_ascii_case("TreeItem"))+ {+ continue;+ }+ let Some(raw) = node_content(tree, node) else {+ continue;+ };+ let Some(mut entry) = parse_fluent_chat_row(raw) else {+ continue;+ };+ entry.node = node;+ entries.push(entry);+ if entries.len() == MAX_STRUCTURAL_CANDIDATES {+ break 'roots;+ }+ }+ }+ if !has_new_message_control || entries.is_empty() {+ return None;+ }++ let mut list = SemanticItem::new(+ "chat-list",+ SemanticKind::Conversation,+ format!("{}:chat-list", profile.id),+ IdentityQuality::Derived,+ );+ list.title = Some(profile.display_name.to_owned());+ list.metadata+ .insert("app".into(), profile.display_name.into());+ list.metadata.insert("family".into(), "conversation".into());+ list.metadata.insert("view".into(), "chat_list".into());+ if let Some(parent) = tree.parent(entries[0].node) {+ list.source_nodes.push(parent);+ }++ let mut items = Vec::with_capacity(entries.len() + 1);+ items.push(list);+ for (index, entry) in entries.into_iter().enumerate() {+ let mut chat = SemanticItem::new(+ format!("chat-{index}"),+ SemanticKind::Conversation,+ format!("{}:chat:{}", profile.id, key_component(&entry.title)),+ IdentityQuality::Derived,+ );+ chat.parent_local_id = Some("chat-list".into());+ chat.title = Some(entry.title);+ chat.body = entry.last_message;+ chat.metadata+ .insert("chat_type".into(), entry.chat_type.into());+ if let Some(sender) = entry.last_sender {+ chat.metadata.insert("last_sender".into(), sender);+ }+ if let Some(last_active) = entry.last_active {+ chat.metadata.insert("last_active".into(), last_active);+ }+ if let Some(status) = entry.status {+ chat.status = Some(status);+ }+ chat.source_nodes.push(entry.node);+ items.push(chat);+ }+ Some(items)+}++fn parse_fluent_chat_row(raw: &str) -> Option<FluentChatListEntry> {+ let raw = raw.trim();+ let (chat_type, row) = raw+ .strip_prefix("Meeting chat ")+ .map(|row| ("meeting", row))+ .or_else(|| raw.strip_prefix("Chat ").map(|row| ("chat", row)))?;+ let (title, detail) = row.split_once(" Last message ")?;+ let title = title.trim();+ if title.is_empty() || title.len() > 240 {+ return None;+ }++ let mut detail = detail.trim();+ let mut last_active = None;+ if let Some((before, token)) = split_last_token(detail) {+ if is_simple_slash_date(token) {+ last_active = Some(token.to_owned());+ detail = before.trim_end();+ }+ }++ let mut status = None;+ if let Some((before, token)) = split_last_token(detail) {+ if matches!(token, "Muted" | "Unmuted" | "Unread") {+ status = Some(token.to_ascii_lowercase());+ detail = before.trim_end();+ }+ }++ let (last_sender, last_message) = match detail.split_once(": ") {+ Some((sender, message))+ if !sender.trim().is_empty()+ && sender.trim().len() <= 80+ && !message.trim().is_empty() =>+ {+ (+ Some(sender.trim().to_owned()),+ Some(message.trim().to_owned()),+ )+ }+ _ if !detail.is_empty() => (None, Some(detail.to_owned())),+ _ => (None, None),+ };++ Some(FluentChatListEntry {+ node: NodeId(0),+ title: title.to_owned(),+ last_sender,+ last_message,+ last_active,+ status,+ chat_type,+ })+}++fn split_last_token(text: &str) -> Option<(&str, &str)> {+ let text = text.trim_end();+ let (before, token) = text.rsplit_once(' ')?;+ let token = token.trim();+ (!token.is_empty()).then_some((before, token))+}++fn is_simple_slash_date(token: &str) -> bool {+ let mut parts = token.split('/');+ let (Some(month), Some(day), Some(year), None) =+ (parts.next(), parts.next(), parts.next(), parts.next())+ else {+ return false;+ };+ let (Ok(month), Ok(day), Ok(year)) = (+ month.parse::<u32>(),+ day.parse::<u32>(),+ year.parse::<u32>(),+ ) else {+ return false;+ };+ (1..=12).contains(&month) && (1..=31).contains(&day) && (2000..=2100).contains(&year)+}++fn tree_any(tree: &SemanticTree, predicate: impl Fn(NodeId) -> bool) -> bool {+ tree.roots()+ .any(|root| tree.descendants(root).any(|node| predicate(node)))+}+/// `collect_text` starting below `root`. A Fluent message container is itself a/// text node holding the flattened turn, so including it would repeat every/// line its children already carry.diff --git a/crates/screenpipe-semantic/src/parsers/native_macos.rs b/crates/screenpipe-semantic/src/parsers/native_macos.rsindex 65c24395d..11624feeb 100644--- a/crates/screenpipe-semantic/src/parsers/native_macos.rs+++ b/crates/screenpipe-semantic/src/parsers/native_macos.rs@@ -690,14 +690,14 @@ fn parse_discord(context: &ParseContext<'_>, tree: &SemanticTree) -> ParseOutcom.collect::<Vec<_>>();let Some(list) = content_lists.into_iter().find(|list| {tree.descendants(*list)- .any(|node| role_is(tree, node, "AXDocumentArticle"))+ .any(|node| is_discord_message_article(tree, node))}) else {return ParseOutcome::NotHandled;};let mut messages = Vec::new();for article in tree.descendants(list)- .filter(|node| role_is(tree, *node, "AXDocumentArticle"))+ .filter(|node| is_discord_message_article(tree, *node)).take(MAX_ITEMS - 1){let (actor, evidence) = direction_actor(tree, article);@@ -725,6 +725,11 @@ fn parse_discord(context: &ParseContext<'_>, tree: &SemanticTree) -> ParseOutcom}}+fn is_discord_message_article(tree: &SemanticTree, node: NodeId) -> bool {+ role_is(tree, node, "AXDocumentArticle")+ || (role_is(tree, node, "AXGroup") && tree.subrole(node) == Some("AXDocument"))+}+fn parse_calendar(tree: &SemanticTree) -> ParseOutcome {let Some(active_view) = find_identifier(tree, "active-view") else {return ParseOutcome::NotHandled;diff --git a/crates/screenpipe-semantic/tests/family_parsers.rs b/crates/screenpipe-semantic/tests/family_parsers.rsindex 36452d1fc..a110c73d2 100644--- a/crates/screenpipe-semantic/tests/family_parsers.rs+++ b/crates/screenpipe-semantic/tests/family_parsers.rs@@ -233,6 +233,50 @@ fn gmail_windows_fixture_extracts_list_rows_with_unread_state() {.all(|row| !row.title.as_deref().unwrap_or_default().contains('\u{a0}')));}+#[test]+fn gmail_macos_web_fixture_extracts_ax_group_list_rows() {+ let source = r#"{+ "app": {+ "platform": "macos",+ "app_id": null,+ "executable": "Google Chrome",+ "display_name": "Chrome",+ "version": null,+ "browser_url": "https://mail.google.com/mail/u/0/#inbox"+ },+ "nodes": [+ { "parent": null, "role": "AXWebArea", "text": "Inbox - Gmail" },+ {+ "parent": 0,+ "role": "AXGroup",+ "text": "unread, Dana Lopez, Parser rollout, 9:14 AM, Please review the parser rollout.",+ "classes": ["zA", "zE"]+ },+ { "parent": 1, "role": "AXStaticText", "text": "Dana Lopez", "classes": ["yX"] },+ {+ "parent": 1,+ "role": "AXStaticText",+ "text": "Parser rollout\u00a0-\u00a0Please review the parser rollout.",+ "classes": ["a4W"]+ },+ { "parent": 1, "role": "AXStaticText", "text": "9:14 AM", "classes": ["xW"] }+ ]+ }"#;+ let items = handled(source, "family.mail");+ assert_eq!(items.len(), 2);+ assert_eq!(+ items[0].metadata.get("view").map(String::as_str),+ Some("list")+ );+ assert_eq!(items[1].actor.as_deref(), Some("Dana Lopez"));+ assert_eq!(items[1].title.as_deref(), Some("Parser rollout"));+ assert_eq!(+ items[1].body.as_deref(),+ Some("Please review the parser rollout.")+ );+ assert_eq!(items[1].status.as_deref(), Some("unread"));+}+#[test]fn chatgpt_windows_fixture_extracts_turns_from_role_description_headings() {let items = handled(@@ -476,6 +520,53 @@ fn teams_windows_fixture_extracts_fluent_control_message_turns() {}}+#[test]+fn teams_windows_chat_list_fixture_extracts_treeitem_rows() {+ let source = r#"{+ "app": {+ "platform": "windows",+ "app_id": "com.microsoft.teams2",+ "executable": "ms-teams.exe",+ "display_name": "Microsoft Teams",+ "version": null,+ "browser_url": null+ },+ "nodes": [+ { "parent": null, "role": "Window", "text": "Microsoft Teams" },+ { "parent": 0, "role": "Button", "text": "New message (Ctrl+N)" },+ {+ "parent": 0,+ "role": "TreeItem",+ "text": "Meeting chat Launch Review Last message You: shipped the parser eval Muted 4/8/2024"+ },+ {+ "parent": 0,+ "role": "TreeItem",+ "text": "Chat Dana Lopez Last message Dana Lopez: numbers are ready 4/9/2024"+ }+ ]+ }"#;+ let items = handled(source, "family.conversation");+ assert_eq!(items.len(), 3, "list plus two chat rows");+ assert_eq!(+ items[0].metadata.get("view").map(String::as_str),+ Some("chat_list")+ );+ assert_eq!(items[1].title.as_deref(), Some("Launch Review"));+ assert_eq!(items[1].body.as_deref(), Some("shipped the parser eval"));+ assert_eq!(items[1].status.as_deref(), Some("muted"));+ assert_eq!(+ items[1].metadata.get("last_sender").map(String::as_str),+ Some("You")+ );+ assert_eq!(+ items[1].metadata.get("last_active").map(String::as_str),+ Some("4/8/2024")+ );+ assert_eq!(items[2].title.as_deref(), Some("Dana Lopez"));+ assert_eq!(items[2].body.as_deref(), Some("numbers are ready"));+}+#[test]fn teams_windows_anchor_requires_the_author_chip() {// Same contract with the author chips replaced by an ordinary actiondiff --git a/crates/screenpipe-semantic/tests/native_macos_parsers.rs b/crates/screenpipe-semantic/tests/native_macos_parsers.rsindex cf2395915..d1664c06b 100644--- a/crates/screenpipe-semantic/tests/native_macos_parsers.rs+++ b/crates/screenpipe-semantic/tests/native_macos_parsers.rs@@ -563,6 +563,72 @@ fn discord_requires_content_list_with_document_articles() {);}+#[test]+fn discord_accepts_document_group_role_variant() {+ let mut builder = SemanticTreeBuilder::new(TreeBudget::default());+ let window = push(+ &mut builder,+ None,+ "AXWindow",+ Some("#release - Discord"),+ None,+ None,+ None,+ None,+ None,+ );+ let list = push(+ &mut builder,+ Some(window),+ "AXList",+ None,+ None,+ None,+ None,+ Some("AXContentList"),+ None,+ );+ let article = push(+ &mut builder,+ Some(list),+ "AXGroup",+ None,+ None,+ None,+ None,+ Some("AXDocument"),+ None,+ );+ push(+ &mut builder,+ Some(article),+ "AXStaticText",+ Some("Dana"),+ None,+ None,+ None,+ None,+ None,+ );+ push(+ &mut builder,+ Some(article),+ "AXStaticText",+ Some("Ship it today."),+ None,+ None,+ None,+ None,+ None,+ );+ let tree = builder.finish();+ let (parser, outcome) = parse("com.hnc.Discord", &tree);+ assert_eq!(parser, "app.macos.discord.content_list");+ let items = handled_items(outcome);+ assert_eq!(items[1].actor.as_deref(), Some("Dana"));+ assert_eq!(items[1].body.as_deref(), Some("Ship it today."));+}+#[test]fn calendar_requires_active_view_and_structured_description() {let mut builder = SemanticTreeBuilder::new(TreeBudget::default());diff --git a/packages/screenpipe-mcp/src/index.ts b/packages/screenpipe-mcp/src/index.tsindex 1b6456a5e..601f349d0 100644--- a/packages/screenpipe-mcp/src/index.ts+++ b/packages/screenpipe-mcp/src/index.ts@@ -1282,6 +1282,7 @@ async function fetchAPI(headers: {"Content-Type": "application/json",...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),+ "x-screenpipe-client": "mcp",...options.headers,},});