Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

feat(semantic): support more mail clients (#6580)

Ezra Ellette8cbfa8cd9481
diff --git a/crates/screenpipe-semantic/src/parsers/catalog.rs b/crates/screenpipe-semantic/src/parsers/catalog.rs
index 25ddd1d68..995fb9445 100644
--- a/crates/screenpipe-semantic/src/parsers/catalog.rs
+++ b/crates/screenpipe-semantic/src/parsers/catalog.rs
@@ -20,7 +20,7 @@ pub enum AppFamily {
/// Public app identity and family membership for one built-in parser profile.
///
-/// The catalog describes 52 supported app targets using public app identities,
+/// The catalog describes 56 supported app targets using public app identities,
/// URL patterns, and stable accessibility contracts.
#[derive(Debug, Clone, Copy)]
pub struct BuiltinAppProfile {
@@ -193,6 +193,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[
&["discord.com/"],
Some("https://discord.com/channels/example/channel"),
),
+ profile(
+ "fastmail",
+ "Fastmail",
+ &[M],
+ &[],
+ &[],
+ &[r"^https?://app\.fastmail\.com/"],
+ &["app.fastmail.com/"],
+ Some("https://app.fastmail.com/mail/Inbox/example"),
+ ),
profile(
"fantastical",
"Fantastical",
@@ -243,6 +253,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[
&["mail.google.com/"],
Some("https://mail.google.com/mail/u/0/#inbox/example"),
),
+ profile(
+ "hey",
+ "HEY",
+ &[M],
+ &[],
+ &[],
+ &[r"^https?://app\.hey\.com/"],
+ &["app.hey.com/"],
+ Some("https://app.hey.com/topics/example"),
+ ),
profile(
"iterm2",
"iTerm2",
@@ -417,6 +437,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[
&[],
None,
),
+ profile(
+ "protonmail",
+ "Proton Mail",
+ &[M],
+ &["ch.protonmail.desktop"],
+ &["Proton Mail", "Proton Mail.exe", "proton-mail"],
+ &[r"^https?://mail\.proton\.me/"],
+ &["mail.proton.me/"],
+ Some("https://mail.proton.me/u/0/inbox/example"),
+ ),
profile(
"slack",
"Slack",
@@ -481,6 +511,16 @@ pub static BUILTIN_APP_PROFILES: &[BuiltinAppProfile] = &[
&[],
None,
),
+ profile(
+ "thunderbird",
+ "Thunderbird",
+ &[M],
+ &["org.mozilla.thunderbird"],
+ &["Thunderbird", "thunderbird.exe", "thunderbird"],
+ &[],
+ &[],
+ None,
+ ),
profile(
"todoist",
"Todoist",
diff --git a/crates/screenpipe-semantic/src/parsers/families.rs b/crates/screenpipe-semantic/src/parsers/families.rs
index 907d6c8d4..cae020039 100644
--- a/crates/screenpipe-semantic/src/parsers/families.rs
+++ b/crates/screenpipe-semantic/src/parsers/families.rs
@@ -92,7 +92,7 @@ impl FamilyParser {
if family == AppFamily::Conversation {
manifest.parser_version = "4".into();
} else if family == AppFamily::Mail {
- manifest.parser_version = "2".into();
+ manifest.parser_version = "3".into();
}
Self { family, manifest }
}
@@ -527,6 +527,9 @@ fn parse_mail(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticI
if profile.id == "gmail" {
return parse_persisted_gmail(profile, tree);
}
+ if profile.id == "protonmail" {
+ return parse_proton_mail(profile, tree);
+ }
return Vec::new();
}
@@ -560,6 +563,97 @@ fn parse_mail(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticI
items
}
+/// Proton Mail exposes each expanded message through stable structural classes:
+/// a `message-header` containing `message-header-recipient-labels`, paired with
+/// a `message-content` body below their nearest shared ancestor. Require that
+/// full trio so mailbox chrome or an empty composer cannot become a message.
+fn parse_proton_mail(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticItem> {
+ let mut messages = Vec::new();
+ let mut message_roots = HashSet::new();
+ 'roots: for root in tree.roots() {
+ for header in tree
+ .descendants(root)
+ .filter(|node| tree.has_class(*node, "message-header"))
+ {
+ let Some(sender_root) = tree
+ .descendants(header)
+ .find(|node| tree.has_class(*node, "message-header-recipient-labels"))
+ else {
+ continue;
+ };
+ let Some(sender) = tree
+ .descendants(sender_root)
+ .find_map(|node| node_content(tree, node))
+ .filter(|sender| sender.len() <= 240)
+ else {
+ continue;
+ };
+
+ let mut ancestor = Some(header);
+ let mut message = None;
+ for _ in 0..=4 {
+ let Some(candidate) = ancestor else {
+ break;
+ };
+ if let Some(body) = first_marked_subtree_text(tree, candidate, &["message-content"])
+ {
+ message = Some((candidate, body));
+ break;
+ }
+ ancestor = tree.parent(candidate);
+ }
+ let Some((message_root, body)) = message else {
+ continue;
+ };
+ if !message_roots.insert(message_root) || body.trim().is_empty() {
+ continue;
+ }
+ messages.push((message_root, sender.to_owned(), body));
+ if messages.len() == MAX_STRUCTURAL_CANDIDATES {
+ break 'roots;
+ }
+ }
+ }
+ if messages.is_empty() {
+ return Vec::new();
+ }
+
+ let subject = first_heading(tree)
+ .unwrap_or(profile.display_name)
+ .to_owned();
+ let mut thread = SemanticItem::new(
+ "thread",
+ SemanticKind::Conversation,
+ format!("{}:mail:{}", profile.id, key_component(&subject)),
+ IdentityQuality::Derived,
+ );
+ thread.title = Some(subject);
+ thread
+ .metadata
+ .insert("app".into(), profile.display_name.into());
+ thread.metadata.insert("family".into(), "mail".into());
+ thread
+ .metadata
+ .insert("surface".into(), "proton_message".into());
+
+ let mut items = Vec::with_capacity(messages.len() + 1);
+ items.push(thread);
+ for (index, (node, sender, body)) in messages.into_iter().enumerate() {
+ let mut message = SemanticItem::new(
+ format!("mail-{index}"),
+ SemanticKind::Message,
+ format!("{}:mail-message:{index}", profile.id),
+ IdentityQuality::Ephemeral,
+ );
+ message.parent_local_id = Some("thread".into());
+ message.actor = Some(sender);
+ message.body = Some(body);
+ message.source_nodes.push(node);
+ items.push(message);
+ }
+ items
+}
+
fn parse_document(profile: &BuiltinAppProfile, tree: &SemanticTree) -> Vec<SemanticItem> {
let mut best: Option<(NodeId, &str, usize)> = None;
for root in tree.roots() {
diff --git a/crates/screenpipe-semantic/tests/builtin_catalog.rs b/crates/screenpipe-semantic/tests/builtin_catalog.rs
index a0fe28086..4b5fe52f0 100644
--- a/crates/screenpipe-semantic/tests/builtin_catalog.rs
+++ b/crates/screenpipe-semantic/tests/builtin_catalog.rs
@@ -25,11 +25,13 @@ const EXPECTED_TARGETS: &[&str] = &[
"codex",
"cursor",
"discord",
+ "fastmail",
"fantastical",
"gemini_desktop",
"geminiweb",
"ghostty",
"gmail",
+ "hey",
"iterm2",
"mail",
"messages",
@@ -45,12 +47,14 @@ const EXPECTED_TARGETS: &[&str] = &[
"obsidian",
"omnifocus",
"pages",
+ "protonmail",
"slack",
"sparkdesktop",
"sparkmailclassic",
"superhuman",
"terminal",
"textedit",
+ "thunderbird",
"todoist",
"toggl",
"vscode",
@@ -84,14 +88,14 @@ fn identity_for(profile: &screenpipe_semantic::parsers::BuiltinAppProfile) -> Ap
}
#[test]
-fn catalog_exactly_matches_all_52_targets() {
+fn catalog_exactly_matches_all_56_targets() {
let actual: BTreeSet<_> = builtin_app_profiles()
.iter()
.map(|profile| profile.id)
.collect();
let expected: BTreeSet<_> = EXPECTED_TARGETS.iter().copied().collect();
- assert_eq!(builtin_app_profiles().len(), 52);
- assert_eq!(actual.len(), 52, "profile ids must be unique");
+ assert_eq!(builtin_app_profiles().len(), 56);
+ assert_eq!(actual.len(), 56, "profile ids must be unique");
assert_eq!(actual, expected);
}
diff --git a/crates/screenpipe-semantic/tests/context_efficiency.rs b/crates/screenpipe-semantic/tests/context_efficiency.rs
index 14b4226bc..f3b10bcd3 100644
--- a/crates/screenpipe-semantic/tests/context_efficiency.rs
+++ b/crates/screenpipe-semantic/tests/context_efficiency.rs
@@ -9,7 +9,7 @@ mod context_eval;
fn semantic_context_is_smaller_than_raw_without_losing_task_facts() {
let report = context_eval::evaluate_suite().expect("context eval must run");
- assert_eq!(report.catalog_profiles, 52);
+ assert_eq!(report.catalog_profiles, 56);
assert_eq!(report.parser_implementations, 23);
assert_eq!(report.representative_cases, 10);
assert_eq!(report.totals.raw_json.offscreen_distractors_retained, 10);
diff --git a/crates/screenpipe-semantic/tests/family_parsers.rs b/crates/screenpipe-semantic/tests/family_parsers.rs
index a110c73d2..07f80d42e 100644
--- a/crates/screenpipe-semantic/tests/family_parsers.rs
+++ b/crates/screenpipe-semantic/tests/family_parsers.rs
@@ -119,6 +119,116 @@ fn mail_fixture_extracts_subject_sender_and_body() {
assert_eq!(items[1].body.as_deref(), Some("Please retry signing."));
}
+#[test]
+fn additional_mail_clients_use_the_structural_family_parser() {
+ let clients = [
+ (
+ "Thunderbird",
+ serde_json::json!({
+ "platform": "windows",
+ "app_id": null,
+ "executable": "thunderbird.exe",
+ "display_name": "Thunderbird",
+ "version": null,
+ "browser_url": null
+ }),
+ ),
+ (
+ "Proton Mail",
+ serde_json::json!({
+ "platform": "macos",
+ "app_id": "ch.protonmail.desktop",
+ "executable": "Proton Mail",
+ "display_name": "Proton Mail",
+ "version": null,
+ "browser_url": null
+ }),
+ ),
+ (
+ "Fastmail",
+ serde_json::json!({
+ "platform": "macos",
+ "app_id": null,
+ "executable": "Safari",
+ "display_name": "Safari",
+ "version": null,
+ "browser_url": "https://app.fastmail.com/mail/Inbox/example"
+ }),
+ ),
+ (
+ "HEY",
+ serde_json::json!({
+ "platform": "windows",
+ "app_id": null,
+ "executable": "chrome.exe",
+ "display_name": "Chrome",
+ "version": null,
+ "browser_url": "https://app.hey.com/topics/example"
+ }),
+ ),
+ ];
+
+ for (client, app) in clients {
+ let source = serde_json::json!({
+ "app": app,
+ "nodes": [
+ { "parent": null, "role": "Window", "title": format!("Parser update - {client}") },
+ { "parent": 0, "role": "Document" },
+ { "parent": 1, "role": "Heading", "classes": ["thread-title"], "title": "Parser update" },
+ { "parent": 1, "role": "Group", "classes": ["email-message"] },
+ { "parent": 3, "role": "Text", "classes": ["sender"], "value": "Dana" },
+ { "parent": 3, "role": "Group", "classes": ["message-body"] },
+ { "parent": 5, "role": "Text", "value": "Please review the mail parser." }
+ ]
+ })
+ .to_string();
+ let items = handled(&source, "family.mail");
+ assert_eq!(items[0].title.as_deref(), Some("Parser update"), "{client}");
+ assert_eq!(items[1].actor.as_deref(), Some("Dana"), "{client}");
+ assert_eq!(
+ items[1].body.as_deref(),
+ Some("Please review the mail parser."),
+ "{client}"
+ );
+ }
+}
+
+#[test]
+fn proton_mail_extracts_its_native_message_structure() {
+ let source = r#"{
+ "app": {
+ "platform": "macos",
+ "app_id": "ch.protonmail.desktop",
+ "executable": "Proton Mail",
+ "display_name": "Proton Mail",
+ "version": null,
+ "browser_url": null
+ },
+ "nodes": [
+ { "parent": null, "role": "AXWebArea" },
+ { "parent": 0, "role": "AXHeading", "text": "Parser update" },
+ { "parent": 0, "role": "AXGroup", "classes": ["message-container"] },
+ { "parent": 2, "role": "AXGroup", "classes": ["message-header", "message-header-collapsed"] },
+ { "parent": 3, "role": "AXGroup", "classes": ["message-header-recipient-labels"] },
+ { "parent": 4, "role": "AXStaticText", "text": "Dana" },
+ { "parent": 2, "role": "AXGroup", "classes": ["message-content"] },
+ { "parent": 6, "role": "AXStaticText", "text": "Please review the Proton parser." }
+ ]
+ }"#;
+ let items = handled(source, "family.mail");
+ assert_eq!(items.len(), 2);
+ assert_eq!(items[0].title.as_deref(), Some("Parser update"));
+ assert_eq!(
+ items[0].metadata.get("surface").map(String::as_str),
+ Some("proton_message")
+ );
+ assert_eq!(items[1].actor.as_deref(), Some("Dana"));
+ assert_eq!(
+ items[1].body.as_deref(),
+ Some("Please review the Proton parser.")
+ );
+}
+
#[test]
fn document_fixture_emits_only_largest_editor_body() {
let items = handled(
diff --git a/docs/SEMANTIC_APP_PARSER_SPEC.md b/docs/SEMANTIC_APP_PARSER_SPEC.md
index 9dcaba0df..b925e2176 100644
--- a/docs/SEMANTIC_APP_PARSER_SPEC.md
+++ b/docs/SEMANTIC_APP_PARSER_SPEC.md
@@ -107,7 +107,7 @@ Screenpipe-owned parser families and exact app overrides.
| Family | Built-in profiles |
|---|---|
| Conversation | Antigravity, Antigravity IDE, ChatGPT, ChatGPT legacy, ChatGPT web, Claude, Claude macOS, ClickUp, ClickUp web, Codex, Cursor, Discord, Gemini desktop, Gemini web, Messages, Messenger, Microsoft Teams, Slack, WhatsApp, WhatsApp web, Windsurf |
-| Mail | Gmail, Mail, Microsoft Outlook, Spark Desktop, Spark Mail Classic, Superhuman |
+| Mail | Fastmail, Gmail, HEY, Mail, Microsoft Outlook, Proton Mail, Spark Desktop, Spark Mail Classic, Superhuman, Thunderbird |
| Editor | Antigravity IDE, Cursor, VS Code, Windsurf, Xcode |
| Document | Antigravity IDE, Claude macOS, ClickUp, ClickUp web, Microsoft Outlook, Microsoft Word, Microsoft Word web, Notes, Notion, Obsidian, Pages, TextEdit, Xcode |
| Task | Antigravity, Asana, Asana web, ClickUp, ClickUp web, Microsoft To Do, OmniFocus, Todoist, Toggl |