Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(semantic): repair priority parser coverage

Louis Beaumont3075996602cf
diff --git a/crates/screenpipe-engine/src/routes/request_origin.rs b/crates/screenpipe-engine/src/routes/request_origin.rs
index 69d79d0db..0303c91ec 100644
--- a/crates/screenpipe-engine/src/routes/request_origin.rs
+++ b/crates/screenpipe-engine/src/routes/request_origin.rs
@@ -6,16 +6,32 @@ use axum::{extract::FromRequestParts, http::request::Parts};
const CLIENT_HEADER: &str = "x-screenpipe-client";
-/// Explicit attribution for user-initiated direct API retrievals.
+/// Privacy-safe attribution for retrieval requests.
///
-/// Missing and unknown values stay unattributed. This prevents app requests,
-/// background Pipes, health checks, and arbitrary user-agent strings from
-/// entering the Repeat Value numerator as direct API use.
-pub struct ExplicitApiClient(bool);
+/// Only a tiny fixed enum is accepted from the client header. Missing and
+/// unknown values are grouped as app-originated requests so arbitrary
+/// user-agent strings never become analytics dimensions, and only the fixed
+/// `api` value enters the Repeat Value direct API numerator.
+pub struct ExplicitApiClient(RequestSource);
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum RequestSource {
+ App,
+ Mcp,
+ Api,
+}
impl ExplicitApiClient {
pub(crate) fn is_direct_api(&self) -> bool {
- self.0
+ self.0 == RequestSource::Api
+ }
+
+ pub(crate) fn source_label(&self) -> &'static str {
+ match self.0 {
+ RequestSource::App => "app",
+ RequestSource::Mcp => "MCP",
+ RequestSource::Api => "API",
+ }
}
}
@@ -33,12 +49,21 @@ impl<S: Send + Sync> FromRequestParts<S> for ExplicitApiClient {
'b: 'c,
{
Box::pin(async move {
- let is_direct_api = parts
+ let source = parts
.headers
.get(CLIENT_HEADER)
.and_then(|value| value.to_str().ok())
- .is_some_and(|value| value == "api");
- Ok(Self(is_direct_api))
+ .map(|value| {
+ if value.eq_ignore_ascii_case("api") {
+ RequestSource::Api
+ } else if value.eq_ignore_ascii_case("mcp") {
+ RequestSource::Mcp
+ } else {
+ RequestSource::App
+ }
+ })
+ .unwrap_or(RequestSource::App);
+ Ok(Self(source))
})
}
}
@@ -65,9 +90,16 @@ mod tests {
#[tokio::test]
async fn accepts_only_the_fixed_direct_api_enum() {
assert!(extract(Some("api")).await.is_direct_api());
+ assert_eq!(extract(Some("api")).await.source_label(), "API");
+ assert_eq!(extract(Some("mcp")).await.source_label(), "MCP");
assert!(!extract(Some("claude-private-project"))
.await
.is_direct_api());
+ assert_eq!(
+ extract(Some("claude-private-project")).await.source_label(),
+ "app"
+ );
assert!(!extract(None).await.is_direct_api());
+ assert_eq!(extract(None).await.source_label(), "app");
}
}
diff --git a/crates/screenpipe-engine/src/routes/search.rs b/crates/screenpipe-engine/src/routes/search.rs
index e3d01cbf7..b57ceb61f 100644
--- a/crates/screenpipe-engine/src/routes/search.rs
+++ b/crates/screenpipe-engine/src/routes/search.rs
@@ -557,6 +557,26 @@ fn capture_direct_api_search_value(client: &ExplicitApiClient, result_count: usi
}
}
+fn parsed_parser_profiles(content_items: &[ContentItem]) -> Vec<String> {
+ let mut parser_ids = Vec::new();
+ for item in content_items {
+ let ContentItem::Parsed(content) = item else {
+ continue;
+ };
+ if parser_ids
+ .iter()
+ .any(|parser_id| parser_id == &content.parser_id)
+ {
+ continue;
+ }
+ parser_ids.push(content.parser_id.clone());
+ if parser_ids.len() == 16 {
+ break;
+ }
+ }
+ parser_ids
+}
+
/// Middle-truncate a string to at most `max_chars` characters.
/// Keeps the first half and last half, inserting a marker in between.
/// Safe on UTF-8 char boundaries.
@@ -1373,15 +1393,26 @@ pub(crate) async fn search(
debug!("search completed: found {} results", total);
+ let response_non_empty = !content_items.is_empty();
+ let parsed_parser_profiles = if parsed_search {
+ parsed_parser_profiles(&content_items)
+ } else {
+ Vec::new()
+ };
+
// Track search analytics
analytics::capture_event_nonblocking(
"search_performed",
serde_json::json!({
"query_length": query.q.as_ref().map(|q| q.len()).unwrap_or(0),
"content_type": format!("{:?}", query.content_type),
+ "request_source": api_client.source_label(),
"has_date_filter": query.start_time.is_some() || query.end_time.is_some(),
"has_app_filter": query.app_name.is_some(),
"result_count": total,
+ "non_empty": response_non_empty,
+ "parsed_non_empty": parsed_search && response_non_empty,
+ "parsed_parser_profiles": parsed_parser_profiles,
"limit": query.pagination.limit,
"offset": query.pagination.offset,
}),
diff --git a/crates/screenpipe-semantic/src/parsers/families.rs b/crates/screenpipe-semantic/src/parsers/families.rs
index 4a4653590..a22e7c17c 100644
--- a/crates/screenpipe-semantic/src/parsers/families.rs
+++ b/crates/screenpipe-semantic/src/parsers/families.rs
@@ -169,6 +169,9 @@ fn parse_conversation(
messages = fluent_control_message_turns(tree);
}
if messages.is_empty() {
+ if let Some(chat_list) = fluent_chat_list(profile, tree) {
+ return chat_list;
+ }
return chromium_chat_list(profile, tree).unwrap_or_default();
}
@@ -562,12 +565,12 @@ fn parse_windows_terminal(profile: &BuiltinAppProfile, tree: &SemanticTree) -> V
vec![terminal]
}
-/// Gmail's thread list through Chromium's UIA bridge on Windows: each
-/// conversation is a `DataItem` row carrying Gmail's long-stable class markers
-/// (`zA` row, `zE` unread) whose child cells are classed `yX` (sender),
-/// `a4W` (subject + snippet), and `xW` (date). AppKit and web-DOM trees never
-/// expose the `DataItem` control type, so this path abstains there. Tokens are
-/// matched with exact `has_class`, never substring signatures — 2-3 char
+/// Gmail's thread list through Chromium's UIA bridge: each conversation row
+/// carries Gmail's long-stable class markers (`zA` row, `zE` unread) whose
+/// child cells are classed `yX` (sender), `a4W` (subject + snippet), and `xW`
+/// (date). Windows publishes these rows as `DataItem`; macOS/AX and browser
+/// bridges can expose the same DOM row as a generic group/row/list item. Tokens
+/// are matched with exact `has_class`, never substring signatures -- 2-3 char
/// tokens would collide with unrelated utility classes.
fn parse_mail_list_rows(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticItem> {
let mut rows = Vec::new();
@@ -644,13 +647,18 @@ fn parse_mail_list_rows(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec
}
fn is_mail_list_row(tree: &SemanticTree, node: NodeId) -> bool {
- tree.role(node)
- .is_some_and(|role| role.eq_ignore_ascii_case("DataItem"))
+ tree.role(node).is_some_and(is_gmail_row_role)
&& tree.has_class(node, "zA")
&& tree.children(node).any(|cell| tree.has_class(cell, "yX"))
&& tree.children(node).any(|cell| tree.has_class(cell, "a4W"))
}
+fn is_gmail_row_role(role: &str) -> bool {
+ ["DataItem", "AXGroup", "AXRow", "Group", "ListItem", "Row"]
+ .iter()
+ .any(|candidate| role.eq_ignore_ascii_case(candidate))
+}
+
fn mail_row_cell_text<'a>(tree: &'a SemanticTree, row: NodeId, class: &str) -> Option<&'a str> {
tree.children(row)
.find(|cell| tree.has_class(*cell, class))
@@ -879,6 +887,186 @@ fn fluent_message_sender<'a>(
})
}
+struct FluentChatListEntry {
+ node: NodeId,
+ title: String,
+ last_sender: Option<String>,
+ last_message: Option<String>,
+ last_active: Option<String>,
+ status: Option<String>,
+ chat_type: &'static str,
+}
+
+fn fluent_chat_list(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Option<Vec<SemanticItem>> {
+ if profile.id != "microsoftteams" || tree_any(tree, |node| is_fluent_message_anchor(tree, node))
+ {
+ return None;
+ }
+
+ let mut has_new_message_control = false;
+ let mut entries = Vec::new();
+ 'roots: for root in tree.roots() {
+ for node in tree.descendants(root) {
+ if button_label(tree, node).is_some_and(|label| {
+ label
+ .trim_start()
+ .get(..11)
+ .is_some_and(|prefix| prefix.eq_ignore_ascii_case("new message"))
+ }) {
+ has_new_message_control = true;
+ continue;
+ }
+ if !tree
+ .role(node)
+ .is_some_and(|role| role.eq_ignore_ascii_case("TreeItem"))
+ {
+ continue;
+ }
+ let Some(raw) = node_content(tree, node) else {
+ continue;
+ };
+ let Some(mut entry) = parse_fluent_chat_row(raw) else {
+ continue;
+ };
+ entry.node = node;
+ entries.push(entry);
+ if entries.len() == MAX_STRUCTURAL_CANDIDATES {
+ break 'roots;
+ }
+ }
+ }
+ if !has_new_message_control || entries.is_empty() {
+ return None;
+ }
+
+ let mut list = SemanticItem::new(
+ "chat-list",
+ SemanticKind::Conversation,
+ format!("{}:chat-list", profile.id),
+ IdentityQuality::Derived,
+ );
+ list.title = Some(profile.display_name.to_owned());
+ list.metadata
+ .insert("app".into(), profile.display_name.into());
+ list.metadata.insert("family".into(), "conversation".into());
+ list.metadata.insert("view".into(), "chat_list".into());
+ if let Some(parent) = tree.parent(entries[0].node) {
+ list.source_nodes.push(parent);
+ }
+
+ let mut items = Vec::with_capacity(entries.len() + 1);
+ items.push(list);
+ for (index, entry) in entries.into_iter().enumerate() {
+ let mut chat = SemanticItem::new(
+ format!("chat-{index}"),
+ SemanticKind::Conversation,
+ format!("{}:chat:{}", profile.id, key_component(&entry.title)),
+ IdentityQuality::Derived,
+ );
+ chat.parent_local_id = Some("chat-list".into());
+ chat.title = Some(entry.title);
+ chat.body = entry.last_message;
+ chat.metadata
+ .insert("chat_type".into(), entry.chat_type.into());
+ if let Some(sender) = entry.last_sender {
+ chat.metadata.insert("last_sender".into(), sender);
+ }
+ if let Some(last_active) = entry.last_active {
+ chat.metadata.insert("last_active".into(), last_active);
+ }
+ if let Some(status) = entry.status {
+ chat.status = Some(status);
+ }
+ chat.source_nodes.push(entry.node);
+ items.push(chat);
+ }
+ Some(items)
+}
+
+fn parse_fluent_chat_row(raw: &str) -> Option<FluentChatListEntry> {
+ let raw = raw.trim();
+ let (chat_type, row) = raw
+ .strip_prefix("Meeting chat ")
+ .map(|row| ("meeting", row))
+ .or_else(|| raw.strip_prefix("Chat ").map(|row| ("chat", row)))?;
+ let (title, detail) = row.split_once(" Last message ")?;
+ let title = title.trim();
+ if title.is_empty() || title.len() > 240 {
+ return None;
+ }
+
+ let mut detail = detail.trim();
+ let mut last_active = None;
+ if let Some((before, token)) = split_last_token(detail) {
+ if is_simple_slash_date(token) {
+ last_active = Some(token.to_owned());
+ detail = before.trim_end();
+ }
+ }
+
+ let mut status = None;
+ if let Some((before, token)) = split_last_token(detail) {
+ if matches!(token, "Muted" | "Unmuted" | "Unread") {
+ status = Some(token.to_ascii_lowercase());
+ detail = before.trim_end();
+ }
+ }
+
+ let (last_sender, last_message) = match detail.split_once(": ") {
+ Some((sender, message))
+ if !sender.trim().is_empty()
+ && sender.trim().len() <= 80
+ && !message.trim().is_empty() =>
+ {
+ (
+ Some(sender.trim().to_owned()),
+ Some(message.trim().to_owned()),
+ )
+ }
+ _ if !detail.is_empty() => (None, Some(detail.to_owned())),
+ _ => (None, None),
+ };
+
+ Some(FluentChatListEntry {
+ node: NodeId(0),
+ title: title.to_owned(),
+ last_sender,
+ last_message,
+ last_active,
+ status,
+ chat_type,
+ })
+}
+
+fn split_last_token(text: &str) -> Option<(&str, &str)> {
+ let text = text.trim_end();
+ let (before, token) = text.rsplit_once(' ')?;
+ let token = token.trim();
+ (!token.is_empty()).then_some((before, token))
+}
+
+fn is_simple_slash_date(token: &str) -> bool {
+ let mut parts = token.split('/');
+ let (Some(month), Some(day), Some(year), None) =
+ (parts.next(), parts.next(), parts.next(), parts.next())
+ else {
+ return false;
+ };
+ let (Ok(month), Ok(day), Ok(year)) = (
+ month.parse::<u32>(),
+ day.parse::<u32>(),
+ year.parse::<u32>(),
+ ) else {
+ return false;
+ };
+ (1..=12).contains(&month) && (1..=31).contains(&day) && (2000..=2100).contains(&year)
+}
+
+fn tree_any(tree: &SemanticTree, predicate: impl Fn(NodeId) -> bool) -> bool {
+ tree.roots()
+ .any(|root| tree.descendants(root).any(|node| predicate(node)))
+}
+
/// `collect_text` starting below `root`. A Fluent message container is itself a
/// text node holding the flattened turn, so including it would repeat every
/// line its children already carry.
diff --git a/crates/screenpipe-semantic/src/parsers/native_macos.rs b/crates/screenpipe-semantic/src/parsers/native_macos.rs
index 65c24395d..11624feeb 100644
--- a/crates/screenpipe-semantic/src/parsers/native_macos.rs
+++ b/crates/screenpipe-semantic/src/parsers/native_macos.rs
@@ -690,14 +690,14 @@ fn parse_discord(context: &ParseContext<'_>, tree: &SemanticTree) -> ParseOutcom
.collect::<Vec<_>>();
let Some(list) = content_lists.into_iter().find(|list| {
tree.descendants(*list)
- .any(|node| role_is(tree, node, "AXDocumentArticle"))
+ .any(|node| is_discord_message_article(tree, node))
}) else {
return ParseOutcome::NotHandled;
};
let mut messages = Vec::new();
for article in tree
.descendants(list)
- .filter(|node| role_is(tree, *node, "AXDocumentArticle"))
+ .filter(|node| is_discord_message_article(tree, *node))
.take(MAX_ITEMS - 1)
{
let (actor, evidence) = direction_actor(tree, article);
@@ -725,6 +725,11 @@ fn parse_discord(context: &ParseContext<'_>, tree: &SemanticTree) -> ParseOutcom
}
}
+fn is_discord_message_article(tree: &SemanticTree, node: NodeId) -> bool {
+ role_is(tree, node, "AXDocumentArticle")
+ || (role_is(tree, node, "AXGroup") && tree.subrole(node) == Some("AXDocument"))
+}
+
fn parse_calendar(tree: &SemanticTree) -> ParseOutcome {
let Some(active_view) = find_identifier(tree, "active-view") else {
return ParseOutcome::NotHandled;
diff --git a/crates/screenpipe-semantic/tests/family_parsers.rs b/crates/screenpipe-semantic/tests/family_parsers.rs
index 36452d1fc..a110c73d2 100644
--- a/crates/screenpipe-semantic/tests/family_parsers.rs
+++ b/crates/screenpipe-semantic/tests/family_parsers.rs
@@ -233,6 +233,50 @@ fn gmail_windows_fixture_extracts_list_rows_with_unread_state() {
.all(|row| !row.title.as_deref().unwrap_or_default().contains('\u{a0}')));
}
+#[test]
+fn gmail_macos_web_fixture_extracts_ax_group_list_rows() {
+ let source = r#"{
+ "app": {
+ "platform": "macos",
+ "app_id": null,
+ "executable": "Google Chrome",
+ "display_name": "Chrome",
+ "version": null,
+ "browser_url": "https://mail.google.com/mail/u/0/#inbox"
+ },
+ "nodes": [
+ { "parent": null, "role": "AXWebArea", "text": "Inbox - Gmail" },
+ {
+ "parent": 0,
+ "role": "AXGroup",
+ "text": "unread, Dana Lopez, Parser rollout, 9:14 AM, Please review the parser rollout.",
+ "classes": ["zA", "zE"]
+ },
+ { "parent": 1, "role": "AXStaticText", "text": "Dana Lopez", "classes": ["yX"] },
+ {
+ "parent": 1,
+ "role": "AXStaticText",
+ "text": "Parser rollout\u00a0-\u00a0Please review the parser rollout.",
+ "classes": ["a4W"]
+ },
+ { "parent": 1, "role": "AXStaticText", "text": "9:14 AM", "classes": ["xW"] }
+ ]
+ }"#;
+ let items = handled(source, "family.mail");
+ assert_eq!(items.len(), 2);
+ assert_eq!(
+ items[0].metadata.get("view").map(String::as_str),
+ Some("list")
+ );
+ assert_eq!(items[1].actor.as_deref(), Some("Dana Lopez"));
+ assert_eq!(items[1].title.as_deref(), Some("Parser rollout"));
+ assert_eq!(
+ items[1].body.as_deref(),
+ Some("Please review the parser rollout.")
+ );
+ assert_eq!(items[1].status.as_deref(), Some("unread"));
+}
+
#[test]
fn chatgpt_windows_fixture_extracts_turns_from_role_description_headings() {
let items = handled(
@@ -476,6 +520,53 @@ fn teams_windows_fixture_extracts_fluent_control_message_turns() {
}
}
+#[test]
+fn teams_windows_chat_list_fixture_extracts_treeitem_rows() {
+ let source = r#"{
+ "app": {
+ "platform": "windows",
+ "app_id": "com.microsoft.teams2",
+ "executable": "ms-teams.exe",
+ "display_name": "Microsoft Teams",
+ "version": null,
+ "browser_url": null
+ },
+ "nodes": [
+ { "parent": null, "role": "Window", "text": "Microsoft Teams" },
+ { "parent": 0, "role": "Button", "text": "New message (Ctrl+N)" },
+ {
+ "parent": 0,
+ "role": "TreeItem",
+ "text": "Meeting chat Launch Review Last message You: shipped the parser eval Muted 4/8/2024"
+ },
+ {
+ "parent": 0,
+ "role": "TreeItem",
+ "text": "Chat Dana Lopez Last message Dana Lopez: numbers are ready 4/9/2024"
+ }
+ ]
+ }"#;
+ let items = handled(source, "family.conversation");
+ assert_eq!(items.len(), 3, "list plus two chat rows");
+ assert_eq!(
+ items[0].metadata.get("view").map(String::as_str),
+ Some("chat_list")
+ );
+ assert_eq!(items[1].title.as_deref(), Some("Launch Review"));
+ assert_eq!(items[1].body.as_deref(), Some("shipped the parser eval"));
+ assert_eq!(items[1].status.as_deref(), Some("muted"));
+ assert_eq!(
+ items[1].metadata.get("last_sender").map(String::as_str),
+ Some("You")
+ );
+ assert_eq!(
+ items[1].metadata.get("last_active").map(String::as_str),
+ Some("4/8/2024")
+ );
+ assert_eq!(items[2].title.as_deref(), Some("Dana Lopez"));
+ assert_eq!(items[2].body.as_deref(), Some("numbers are ready"));
+}
+
#[test]
fn teams_windows_anchor_requires_the_author_chip() {
// Same contract with the author chips replaced by an ordinary action
diff --git a/crates/screenpipe-semantic/tests/native_macos_parsers.rs b/crates/screenpipe-semantic/tests/native_macos_parsers.rs
index cf2395915..d1664c06b 100644
--- a/crates/screenpipe-semantic/tests/native_macos_parsers.rs
+++ b/crates/screenpipe-semantic/tests/native_macos_parsers.rs
@@ -563,6 +563,72 @@ fn discord_requires_content_list_with_document_articles() {
);
}
+#[test]
+fn discord_accepts_document_group_role_variant() {
+ let mut builder = SemanticTreeBuilder::new(TreeBudget::default());
+ let window = push(
+ &mut builder,
+ None,
+ "AXWindow",
+ Some("#release - Discord"),
+ None,
+ None,
+ None,
+ None,
+ None,
+ );
+ let list = push(
+ &mut builder,
+ Some(window),
+ "AXList",
+ None,
+ None,
+ None,
+ None,
+ Some("AXContentList"),
+ None,
+ );
+ let article = push(
+ &mut builder,
+ Some(list),
+ "AXGroup",
+ None,
+ None,
+ None,
+ None,
+ Some("AXDocument"),
+ None,
+ );
+ push(
+ &mut builder,
+ Some(article),
+ "AXStaticText",
+ Some("Dana"),
+ None,
+ None,
+ None,
+ None,
+ None,
+ );
+ push(
+ &mut builder,
+ Some(article),
+ "AXStaticText",
+ Some("Ship it today."),
+ None,
+ None,
+ None,
+ None,
+ None,
+ );
+ let tree = builder.finish();
+ let (parser, outcome) = parse("com.hnc.Discord", &tree);
+ assert_eq!(parser, "app.macos.discord.content_list");
+ let items = handled_items(outcome);
+ assert_eq!(items[1].actor.as_deref(), Some("Dana"));
+ assert_eq!(items[1].body.as_deref(), Some("Ship it today."));
+}
+
#[test]
fn calendar_requires_active_view_and_structured_description() {
let mut builder = SemanticTreeBuilder::new(TreeBudget::default());
diff --git a/packages/screenpipe-mcp/src/index.ts b/packages/screenpipe-mcp/src/index.ts
index 1b6456a5e..601f349d0 100644
--- a/packages/screenpipe-mcp/src/index.ts
+++ b/packages/screenpipe-mcp/src/index.ts
@@ -1282,6 +1282,7 @@ async function fetchAPI(
headers: {
"Content-Type": "application/json",
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
+ "x-screenpipe-client": "mcp",
...options.headers,
},
});