YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat(semantic): support more mail clients (#6580)
Ezra Ellette8cbfa8cd9481
diff --git a/crates/screenpipe-semantic/src/parsers/catalog.rs b/crates/screenpipe-semantic/src/parsers/catalog.rsindex 25ddd1d68..995fb9445 100644--- a/crates/screenpipe-semantic/src/parsers/catalog.rs+++ b/crates/screenpipe-semantic/src/parsers/catalog.rs@@ -20,7 +20,7 @@ pub enum AppFamily {/// Public app identity and family membership for one built-in parser profile.///-/// The catalog describes 52 supported app targets using public app identities,+/// The catalog describes 56 supported app targets using public app identities,/// URL patterns, and stable accessibility contracts.#[derive(Debug, Clone, Copy)]pub struct BuiltinAppProfile {@@ -193,6 +193,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&["discord.com/"],Some("https://discord.com/channels/example/channel"),),+ profile(+ "fastmail",+ "Fastmail",+ &[M],+ &[],+ &[],+ &[r"^https?://app\.fastmail\.com/"],+ &["app.fastmail.com/"],+ Some("https://app.fastmail.com/mail/Inbox/example"),+ ),profile("fantastical","Fantastical",@@ -243,6 +253,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&["mail.google.com/"],Some("https://mail.google.com/mail/u/0/#inbox/example"),),+ profile(+ "hey",+ "HEY",+ &[M],+ &[],+ &[],+ &[r"^https?://app\.hey\.com/"],+ &["app.hey.com/"],+ Some("https://app.hey.com/topics/example"),+ ),profile("iterm2","iTerm2",@@ -417,6 +437,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&[],None,),+ profile(+ "protonmail",+ "Proton Mail",+ &[M],+ &["ch.protonmail.desktop"],+ &["Proton Mail", "Proton Mail.exe", "proton-mail"],+ &[r"^https?://mail\.proton\.me/"],+ &["mail.proton.me/"],+ Some("https://mail.proton.me/u/0/inbox/example"),+ ),profile("slack","Slack",@@ -481,6 +511,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[&[],None,),+ profile(+ "thunderbird",+ "Thunderbird",+ &[M],+ &["org.mozilla.thunderbird"],+ &["Thunderbird", "thunderbird.exe", "thunderbird"],+ &[],+ &[],+ None,+ ),profile("todoist","Todoist",diff --git a/crates/screenpipe-semantic/src/parsers/families.rs b/crates/screenpipe-semantic/src/parsers/families.rsindex 907d6c8d4..cae020039 100644--- a/crates/screenpipe-semantic/src/parsers/families.rs+++ b/crates/screenpipe-semantic/src/parsers/families.rs@@ -92,7 +92,7 @@ impl FamilyParser {if family == AppFamily::Conversation {manifest.parser_version = "4".into();} else if family == AppFamily::Mail {- manifest.parser_version = "2".into();+ manifest.parser_version = "3".into();}Self { family, manifest }}@@ -527,6 +527,9 @@ fn parse_mail(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticIif profile.id == "gmail" {return parse_persisted_gmail(profile, tree);}+ if profile.id == "protonmail" {+ return parse_proton_mail(profile, tree);+ }return Vec::new();}@@ -560,6 +563,97 @@ fn parse_mail(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticIitems}+/// Proton Mail exposes each expanded message through stable structural classes:+/// a `message-header` containing `message-header-recipient-labels`, paired with+/// a `message-content` body below their nearest shared ancestor. Require that+/// full trio so mailbox chrome or an empty composer cannot become a message.+fn parse_proton_mail(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticItem> {+ let mut messages = Vec::new();+ let mut message_roots = HashSet::new();+ 'roots: for root in tree.roots() {+ for header in tree+ .descendants(root)+ .filter(|node| tree.has_class(*node, "message-header"))+ {+ let Some(sender_root) = tree+ .descendants(header)+ .find(|node| tree.has_class(*node, "message-header-recipient-labels"))+ else {+ continue;+ };+ let Some(sender) = tree+ .descendants(sender_root)+ .find_map(|node| node_content(tree, node))+ .filter(|sender| sender.len() <= 240)+ else {+ continue;+ };++ let mut ancestor = Some(header);+ let mut message = None;+ for _ in 0..=4 {+ let Some(candidate) = ancestor else {+ break;+ };+ if let Some(body) = first_marked_subtree_text(tree, candidate, &["message-content"])+ {+ message = Some((candidate, body));+ break;+ }+ ancestor = tree.parent(candidate);+ }+ let Some((message_root, body)) = message else {+ continue;+ };+ if !message_roots.insert(message_root) || body.trim().is_empty() {+ continue;+ }+ messages.push((message_root, sender.to_owned(), body));+ if messages.len() == MAX_STRUCTURAL_CANDIDATES {+ break 'roots;+ }+ }+ }+ if messages.is_empty() {+ return Vec::new();+ }++ let subject = first_heading(tree)+ .unwrap_or(profile.display_name)+ .to_owned();+ let mut thread = SemanticItem::new(+ "thread",+ SemanticKind::Conversation,+ format!("{}:mail:{}", profile.id, key_component(&subject)),+ IdentityQuality::Derived,+ );+ thread.title = Some(subject);+ thread+ .metadata+ .insert("app".into(), profile.display_name.into());+ thread.metadata.insert("family".into(), "mail".into());+ thread+ .metadata+ .insert("surface".into(), "proton_message".into());++ let mut items = Vec::with_capacity(messages.len() + 1);+ items.push(thread);+ for (index, (node, sender, body)) in messages.into_iter().enumerate() {+ let mut message = SemanticItem::new(+ format!("mail-{index}"),+ SemanticKind::Message,+ format!("{}:mail-message:{index}", profile.id),+ IdentityQuality::Ephemeral,+ );+ message.parent_local_id = Some("thread".into());+ message.actor = Some(sender);+ message.body = Some(body);+ message.source_nodes.push(node);+ items.push(message);+ }+ items+}+fn parse_document(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticItem> {let mut best: Option<(NodeId, &str, usize)> = None;for root in tree.roots() {diff --git a/crates/screenpipe-semantic/tests/builtin_catalog.rs b/crates/screenpipe-semantic/tests/builtin_catalog.rsindex a0fe28086..4b5fe52f0 100644--- a/crates/screenpipe-semantic/tests/builtin_catalog.rs+++ b/crates/screenpipe-semantic/tests/builtin_catalog.rs@@ -25,11 +25,13 @@ const EXPECTED_TARGETS: &[&str] = &["codex","cursor","discord",+ "fastmail","fantastical","gemini_desktop","geminiweb","ghostty","gmail",+ "hey","iterm2","mail","messages",@@ -45,12 +47,14 @@ const EXPECTED_TARGETS: &[&str] = &["obsidian","omnifocus","pages",+ "protonmail","slack","sparkdesktop","sparkmailclassic","superhuman","terminal","textedit",+ "thunderbird","todoist","toggl","vscode",@@ -84,14 +88,14 @@ fn identity_for(profile: &screenpipe_semantic::parsers::BuiltinAppProfile) -> Ap}#[test]-fn catalog_exactly_matches_all_52_targets() {+fn catalog_exactly_matches_all_56_targets() {let actual: BTreeSet<_> = builtin_app_profiles().iter().map(|profile| profile.id).collect();let expected: BTreeSet<_> = EXPECTED_TARGETS.iter().copied().collect();- assert_eq!(builtin_app_profiles().len(), 52);- assert_eq!(actual.len(), 52, "profile ids must be unique");+ assert_eq!(builtin_app_profiles().len(), 56);+ assert_eq!(actual.len(), 56, "profile ids must be unique");assert_eq!(actual, expected);}diff --git a/crates/screenpipe-semantic/tests/context_efficiency.rs b/crates/screenpipe-semantic/tests/context_efficiency.rsindex 14b4226bc..f3b10bcd3 100644--- a/crates/screenpipe-semantic/tests/context_efficiency.rs+++ b/crates/screenpipe-semantic/tests/context_efficiency.rs@@ -9,7 +9,7 @@ mod context_eval;fn semantic_context_is_smaller_than_raw_without_losing_task_facts() {let report = context_eval::evaluate_suite().expect("context eval must run");- assert_eq!(report.catalog_profiles, 52);+ assert_eq!(report.catalog_profiles, 56);assert_eq!(report.parser_implementations, 23);assert_eq!(report.representative_cases, 10);assert_eq!(report.totals.raw_json.offscreen_distractors_retained, 10);diff --git a/crates/screenpipe-semantic/tests/family_parsers.rs b/crates/screenpipe-semantic/tests/family_parsers.rsindex a110c73d2..07f80d42e 100644--- a/crates/screenpipe-semantic/tests/family_parsers.rs+++ b/crates/screenpipe-semantic/tests/family_parsers.rs@@ -119,6 +119,116 @@ fn mail_fixture_extracts_subject_sender_and_body() {assert_eq!(items[1].body.as_deref(), Some("Please retry signing."));}+#[test]+fn additional_mail_clients_use_the_structural_family_parser() {+ let clients = [+ (+ "Thunderbird",+ serde_json::json!({+ "platform": "windows",+ "app_id": null,+ "executable": "thunderbird.exe",+ "display_name": "Thunderbird",+ "version": null,+ "browser_url": null+ }),+ ),+ (+ "Proton Mail",+ serde_json::json!({+ "platform": "macos",+ "app_id": "ch.protonmail.desktop",+ "executable": "Proton Mail",+ "display_name": "Proton Mail",+ "version": null,+ "browser_url": null+ }),+ ),+ (+ "Fastmail",+ serde_json::json!({+ "platform": "macos",+ "app_id": null,+ "executable": "Safari",+ "display_name": "Safari",+ "version": null,+ "browser_url": "https://app.fastmail.com/mail/Inbox/example"+ }),+ ),+ (+ "HEY",+ serde_json::json!({+ "platform": "windows",+ "app_id": null,+ "executable": "chrome.exe",+ "display_name": "Chrome",+ "version": null,+ "browser_url": "https://app.hey.com/topics/example"+ }),+ ),+ ];++ for (client, app) in clients {+ let source = serde_json::json!({+ "app": app,+ "nodes": [+ { "parent": null, "role": "Window", "title": format!("Parser update - {client}") },+ { "parent": 0, "role": "Document" },+ { "parent": 1, "role": "Heading", "classes": ["thread-title"], "title": "Parser update" },+ { "parent": 1, "role": "Group", "classes": ["email-message"] },+ { "parent": 3, "role": "Text", "classes": ["sender"], "value": "Dana" },+ { "parent": 3, "role": "Group", "classes": ["message-body"] },+ { "parent": 5, "role": "Text", "value": "Please review the mail parser." }+ ]+ })+ .to_string();+ let items = handled(&source, "family.mail");+ assert_eq!(items[0].title.as_deref(), Some("Parser update"), "{client}");+ assert_eq!(items[1].actor.as_deref(), Some("Dana"), "{client}");+ assert_eq!(+ items[1].body.as_deref(),+ Some("Please review the mail parser."),+ "{client}"+ );+ }+}++#[test]+fn proton_mail_extracts_its_native_message_structure() {+ let source = r#"{+ "app": {+ "platform": "macos",+ "app_id": "ch.protonmail.desktop",+ "executable": "Proton Mail",+ "display_name": "Proton Mail",+ "version": null,+ "browser_url": null+ },+ "nodes": [+ { "parent": null, "role": "AXWebArea" },+ { "parent": 0, "role": "AXHeading", "text": "Parser update" },+ { "parent": 0, "role": "AXGroup", "classes": ["message-container"] },+ { "parent": 2, "role": "AXGroup", "classes": ["message-header", "message-header-collapsed"] },+ { "parent": 3, "role": "AXGroup", "classes": ["message-header-recipient-labels"] },+ { "parent": 4, "role": "AXStaticText", "text": "Dana" },+ { "parent": 2, "role": "AXGroup", "classes": ["message-content"] },+ { "parent": 6, "role": "AXStaticText", "text": "Please review the Proton parser." }+ ]+ }"#;+ let items = handled(source, "family.mail");+ assert_eq!(items.len(), 2);+ assert_eq!(items[0].title.as_deref(), Some("Parser update"));+ assert_eq!(+ items[0].metadata.get("surface").map(String::as_str),+ Some("proton_message")+ );+ assert_eq!(items[1].actor.as_deref(), Some("Dana"));+ assert_eq!(+ items[1].body.as_deref(),+ Some("Please review the Proton parser.")+ );+}+#[test]fn document_fixture_emits_only_largest_editor_body() {let items = handled(diff --git a/docs/SEMANTIC_APP_PARSER_SPEC.md b/docs/SEMANTIC_APP_PARSER_SPEC.mdindex 9dcaba0df..b925e2176 100644--- a/docs/SEMANTIC_APP_PARSER_SPEC.md+++ b/docs/SEMANTIC_APP_PARSER_SPEC.md@@ -107,7 +107,7 @@ Screenpipe-owned parser families and exact app overrides.| Family | Built-in profiles ||---|---|| Conversation | Antigravity, Antigravity IDE, ChatGPT, ChatGPT legacy, ChatGPT web, Claude, Claude macOS, ClickUp, ClickUp web, Codex, Cursor, Discord, Gemini desktop, Gemini web, Messages, Messenger, Microsoft Teams, Slack, WhatsApp, WhatsApp web, Windsurf |-| Mail | Gmail, Mail, Microsoft Outlook, Spark Desktop, Spark Mail Classic, Superhuman |+| Mail | Fastmail, Gmail, HEY, Mail, Microsoft Outlook, Proton Mail, Spark Desktop, Spark Mail Classic, Superhuman, Thunderbird || Editor | Antigravity IDE, Cursor, VS Code, Windsurf, Xcode || Document | Antigravity IDE, Claude macOS, ClickUp, ClickUp web, Microsoft Outlook, Microsoft Word, Microsoft Word web, Notes, Notion, Obsidian, Pages, TextEdit, Xcode || Task | Antigravity, Asana, Asana web, ClickUp, ClickUp web, Microsoft To Do, OmniFocus, Todoist, Toggl |