Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(email): denormalize thread calendar-attachment flag (#4515)
Evan Hutnik44daee41d535
diff --git a/rust/cloud-storage/.sqlx/query-2810a4a2c8ea8504b36826226c558f1bde9259ab8a5dd1fd9556e43e2fa6513e.json b/rust/cloud-storage/.sqlx/query-2810a4a2c8ea8504b36826226c558f1bde9259ab8a5dd1fd9556e43e2fa6513e.jsonnew file mode 100644index 0000000000..0defaa5187--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-2810a4a2c8ea8504b36826226c558f1bde9259ab8a5dd1fd9556e43e2fa6513e.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE email_threads t\n SET has_calendar_attachment = calc.has_cal\n FROM (\n SELECT EXISTS (\n SELECT 1\n FROM email_messages m\n JOIN email_attachments a ON a.message_id = m.id\n WHERE m.thread_id = $1\n AND (a.filename ILIKE '%.ics'\n OR a.mime_type = 'text/calendar'\n OR a.mime_type = 'application/ics')\n ) AS has_cal\n ) calc\n WHERE t.id = $1\n AND t.has_calendar_attachment IS DISTINCT FROM calc.has_cal\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "2810a4a2c8ea8504b36826226c558f1bde9259ab8a5dd1fd9556e43e2fa6513e"+}diff --git a/rust/cloud-storage/.sqlx/query-76ee78dac93f9184df7cd719f7b2b52709675e92466ec5907f1ded5bac7b21f2.json b/rust/cloud-storage/.sqlx/query-76ee78dac93f9184df7cd719f7b2b52709675e92466ec5907f1ded5bac7b21f2.jsonnew file mode 100644index 0000000000..073e543bbd--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-76ee78dac93f9184df7cd719f7b2b52709675e92466ec5907f1ded5bac7b21f2.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE email_threads t\n SET has_calendar_attachment = true\n FROM (\n SELECT DISTINCT m.thread_id\n FROM email_messages m\n JOIN email_attachments a ON a.message_id = m.id\n WHERE m.link_id = $1\n AND (a.filename ILIKE '%.ics'\n OR a.mime_type = 'text/calendar'\n OR a.mime_type = 'application/ics')\n ) cal\n WHERE t.id = cal.thread_id\n AND NOT t.has_calendar_attachment\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "76ee78dac93f9184df7cd719f7b2b52709675e92466ec5907f1ded5bac7b21f2"+}diff --git a/rust/cloud-storage/.sqlx/query-855755b529207245d323d939a9d1718c12a6509c805f59bff9997e661f47142f.json b/rust/cloud-storage/.sqlx/query-855755b529207245d323d939a9d1718c12a6509c805f59bff9997e661f47142f.jsonnew file mode 100644index 0000000000..64f2b0796a--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-855755b529207245d323d939a9d1718c12a6509c805f59bff9997e661f47142f.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT thread_id FROM email_messages WHERE id = $1",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "thread_id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "855755b529207245d323d939a9d1718c12a6509c805f59bff9997e661f47142f"+}diff --git a/rust/cloud-storage/.sqlx/query-b0aa946e4aba01ed02a9f633f563437e88be9f676e95ec13725bc598614bf24c.json b/rust/cloud-storage/.sqlx/query-b0aa946e4aba01ed02a9f633f563437e88be9f676e95ec13725bc598614bf24c.jsonnew file mode 100644index 0000000000..3ff7622ee4--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-b0aa946e4aba01ed02a9f633f563437e88be9f676e95ec13725bc598614bf24c.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT id FROM email_links WHERE macro_id = $1",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Text"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "b0aa946e4aba01ed02a9f633f563437e88be9f676e95ec13725bc598614bf24c"+}diff --git a/rust/cloud-storage/.sqlx/query-b84715ccb1c38d74d76b11969b595a59db45d27a49f994583c9619b1a1df4fa3.json b/rust/cloud-storage/.sqlx/query-b84715ccb1c38d74d76b11969b595a59db45d27a49f994583c9619b1a1df4fa3.jsonnew file mode 100644index 0000000000..144d55d4c0--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-b84715ccb1c38d74d76b11969b595a59db45d27a49f994583c9619b1a1df4fa3.json@@ -0,0 +1,20 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT DISTINCT macro_id FROM email_links ORDER BY macro_id",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "macro_id",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ false+ ]+ },+ "hash": "b84715ccb1c38d74d76b11969b595a59db45d27a49f994583c9619b1a1df4fa3"+}diff --git a/rust/cloud-storage/.sqlx/query-e802b95145183d9bec6fb5f542fe93d392fdcf0fbebc9935f263f3f4fd1d8c34.json b/rust/cloud-storage/.sqlx/query-e802b95145183d9bec6fb5f542fe93d392fdcf0fbebc9935f263f3f4fd1d8c34.jsondeleted file mode 100644index e30af0774f..0000000000--- a/rust/cloud-storage/.sqlx/query-e802b95145183d9bec6fb5f542fe93d392fdcf0fbebc9935f263f3f4fd1d8c34.json+++ /dev/null@@ -1,128 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n SELECT\n t.id,\n t.provider_id,\n t.inbox_visible,\n t.is_read,\n t.effective_ts AS \"sort_ts!\",\n t.created_at AS \"created_at!\",\n t.updated_at AS \"updated_at!\",\n t.project_id,\n t.viewed_at AS \"viewed_at?\",\n lmp.subject AS \"name?\",\n lmp.snippet AS \"snippet?\",\n lmp.is_draft,\n (\n SELECT EXISTS (\n SELECT 1\n FROM email_messages m_imp\n JOIN email_message_labels ml ON m_imp.id = ml.message_id\n JOIN email_labels l ON ml.label_id = l.id\n WHERE m_imp.thread_id = t.id\n AND l.name = 'IMPORTANT'\n AND l.link_id = t.link_id\n )\n ) AS \"is_important!\",\n c.email_address AS \"sender_email?\",\n COALESCE(lmp.from_name, c.name) AS \"sender_name?\",\n c.sfs_photo_url as \"sender_photo_url?\",\n el.macro_id AS \"owner_id!\",\n el.id AS \"link_id!\"\n FROM (\n -- Step 1: Efficiently find and sort ONLY the top N+1 candidate threads.\n -- This subquery only touches `threads` and `user_history`.\n SELECT\n t.id,\n t.provider_id,\n t.link_id,\n t.inbox_visible,\n t.is_read,\n t.project_id,\n t.latest_outbound_message_ts AS created_at,\n t.latest_outbound_message_ts AS updated_at,\n uh.updated_at AS viewed_at,\n CASE $5 -- sort_method_str\n WHEN 'viewed_at' THEN COALESCE(uh.\"updated_at\", '1970-01-01 00:00:00+00')\n WHEN 'viewed_updated' THEN COALESCE(uh.updated_at, t.latest_outbound_message_ts)\n ELSE t.latest_outbound_message_ts\n END AS effective_ts\n FROM email_threads t\n LEFT JOIN email_user_history uh ON uh.thread_id = t.id AND uh.link_id = t.link_id\n WHERE\n t.link_id = ANY($1)\n AND t.latest_outbound_message_ts IS NOT NULL\n \n -- Cursor logic moved inside for maximum efficiency\n AND (($3::timestamptz IS NULL) OR (\n -- This CASE must exactly match the one that defines `effective_ts`\n CASE $5 -- sort_method_str\n WHEN 'viewed_at' THEN COALESCE(uh.\"updated_at\", '1970-01-01 00:00:00+00')\n WHEN 'viewed_updated' THEN COALESCE(uh.updated_at, t.latest_outbound_message_ts)\n ELSE t.latest_outbound_message_ts\n END, t.id\n ) < ($3::timestamptz, $4::uuid))\n ORDER BY effective_ts DESC, t.updated_at DESC\n LIMIT $2\n ) AS t\n -- Step 2: For EACH of the limited threads from above, find its latest SENT, non-trashed message.\n CROSS JOIN LATERAL (\n SELECT\n m.subject,\n m.snippet,\n m.from_contact_id,\n m.from_name,\n m.is_draft\n FROM email_messages m\n WHERE m.thread_id = t.id\n AND m.is_sent = TRUE -- This condition is specific to the \"Sent\" view\n AND NOT EXISTS (\n SELECT 1 FROM email_message_labels ml JOIN email_labels l ON ml.label_id = l.id\n WHERE ml.message_id = m.id AND l.name = 'TRASH' AND l.link_id = t.link_id\n )\n ORDER BY m.internal_date_ts DESC\n LIMIT 1\n ) AS lmp\n -- Step 3: Join to get the sender's details for the final result set.\n LEFT JOIN email_contacts c ON lmp.from_contact_id = c.id\n JOIN email_links el ON t.link_id = el.id\n -- Final ordering is preserved because the input `t` is already sorted.\n ORDER BY t.effective_ts DESC, t.updated_at DESC\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "id",- "type_info": "Uuid"- },- {- "ordinal": 1,- "name": "provider_id",- "type_info": "Text"- },- {- "ordinal": 2,- "name": "inbox_visible",- "type_info": "Bool"- },- {- "ordinal": 3,- "name": "is_read",- "type_info": "Bool"- },- {- "ordinal": 4,- "name": "sort_ts!",- "type_info": "Timestamptz"- },- {- "ordinal": 5,- "name": "created_at!",- "type_info": "Timestamptz"- },- {- "ordinal": 6,- "name": "updated_at!",- "type_info": "Timestamptz"- },- {- "ordinal": 7,- "name": "project_id",- "type_info": "Text"- },- {- "ordinal": 8,- "name": "viewed_at?",- "type_info": "Timestamptz"- },- {- "ordinal": 9,- "name": "name?",- "type_info": "Text"- },- {- "ordinal": 10,- "name": "snippet?",- "type_info": "Text"- },- {- "ordinal": 11,- "name": "is_draft",- "type_info": "Bool"- },- {- "ordinal": 12,- "name": "is_important!",- "type_info": "Bool"- },- {- "ordinal": 13,- "name": "sender_email?",- "type_info": "Varchar"- },- {- "ordinal": 14,- "name": "sender_name?",- "type_info": "Varchar"- },- {- "ordinal": 15,- "name": "sender_photo_url?",- "type_info": "Text"- },- {- "ordinal": 16,- "name": "owner_id!",- "type_info": "Text"- },- {- "ordinal": 17,- "name": "link_id!",- "type_info": "Uuid"- }- ],- "parameters": {- "Left": [- "UuidArray",- "Int8",- "Timestamptz",- "Uuid",- "Text"- ]- },- "nullable": [- false,- true,- false,- false,- null,- true,- true,- true,- false,- true,- true,- false,- null,- false,- null,- true,- false,- false- ]- },- "hash": "e802b95145183d9bec6fb5f542fe93d392fdcf0fbebc9935f263f3f4fd1d8c34"-}diff --git a/rust/cloud-storage/email/fixtures/email_dynamic_query_calendar.sql b/rust/cloud-storage/email/fixtures/email_dynamic_query_calendar.sqlindex 002c766c14..ee69181325 100644--- a/rust/cloud-storage/email/fixtures/email_dynamic_query_calendar.sql+++ b/rust/cloud-storage/email/fixtures/email_dynamic_query_calendar.sql@@ -33,3 +33,16 @@ VALUES-- Thread 7 (inbox with CC): only a non-calendar attachment → should NOT match('90000007-0000-0000-0000-000000000007', '30000007-0000-0000-0000-000000000007','report.pdf', 'application/pdf', NOW());++-- Maintain the denormalized flag the CalendarOnly filter reads (set at+-- attachment ingest in production; mirrored here for the raw inserts above).+UPDATE email_threads t+SET has_calendar_attachment = true+WHERE t.id IN (+ SELECT m.thread_id+ FROM email_attachments a+ JOIN email_messages m ON m.id = a.message_id+ WHERE a.filename ILIKE '%.ics'+ OR a.mime_type = 'text/calendar'+ OR a.mime_type = 'application/ics'+);diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rsindex bed2d12f3e..4bcebb70c3 100644--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs@@ -761,19 +761,11 @@ pub(super) fn build_thread_email_filter(f}- filter_ast::ExprFrame::Literal(EmailLiteral::CalendarOnly(true)) => SqlFragment::raw(- r#"EXISTS (- SELECT 1- FROM email_messages m_cal- JOIN email_attachments a_cal ON a_cal.message_id = m_cal.id- WHERE m_cal.thread_id = t.id- AND (- a_cal.filename ILIKE '%.ics'- OR a_cal.mime_type = 'text/calendar'- OR a_cal.mime_type = 'application/ics'- )- )"#,- ),+ // Denormalized flag maintained at attachment ingest — deriving this+ // from email_attachments at query time is prohibitively slow.+ filter_ast::ExprFrame::Literal(EmailLiteral::CalendarOnly(true)) => {+ SqlFragment::raw("t.has_calendar_attachment")+ }filter_ast::ExprFrame::Literal(EmailLiteral::CalendarOnly(false)) => {SqlFragment::raw("TRUE")diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rsindex da2c309b43..92f7684821 100644--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs@@ -1229,17 +1229,34 @@ fn test_full_query_emits_matching_threads_cte_and_in_reference() {}#[test]-fn test_build_thread_email_filter_calendar_only_true_emits_ics_exists() {+fn test_build_thread_email_filter_calendar_only_true_uses_thread_flag() {let expr = Expr::Literal(EmailLiteral::CalendarOnly(true));let result = build_thread_email_filter(&expr, DEFAULT_SORT_TS);let debug = result.to_debug_sql();- assert!(debug.contains("EXISTS"));- assert!(debug.contains("email_attachments"));- assert!(debug.contains("m_cal.thread_id = t.id"));- assert!(debug.contains("a_cal.filename ILIKE '%.ics'"));- assert!(debug.contains("a_cal.mime_type = 'text/calendar'"));- assert!(debug.contains("a_cal.mime_type = 'application/ics'"));+ assert!(debug.contains("t.has_calendar_attachment"));+ // No per-thread attachment probe remains in the candidate WHERE.+ assert!(!debug.contains("email_attachments"));+}++#[test]+fn test_full_query_calendar_only_uses_thread_flag_without_cte() {+ let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);+ let expr = Expr::Literal(EmailLiteral::CalendarOnly(true));+ let sql = super::query::debug_build_query_sql(&view, &expr);++ assert!(+ sql.contains("t.has_calendar_attachment"),+ "candidate WHERE must use the denormalized flag: {sql}"+ );+ assert!(+ !sql.contains("calendar_threads"),+ "calendar CTE should be gone: {sql}"+ );+ assert!(+ !sql.contains("email_attachments"),+ "no attachment scan should remain: {sql}"+ );}#[test]diff --git a/rust/cloud-storage/email_db_client/fixtures/sync_thread_calendar_flag.sql b/rust/cloud-storage/email_db_client/fixtures/sync_thread_calendar_flag.sqlnew file mode 100644index 0000000000..7af259e183--- /dev/null+++ b/rust/cloud-storage/email_db_client/fixtures/sync_thread_calendar_flag.sql@@ -0,0 +1,68 @@+-- Fixture for the denormalized email_threads.has_calendar_attachment flag.+--+-- Thread 1 (flag=false): message b501 has an .ics attachment, message b502+-- has none — sync should set the flag true; deleting b501 should clear it.+-- Thread 2 (flag=true, stale on purpose): message b503 has only a PDF —+-- sync should clear the flag.+-- Thread 3 (flag=false): message b504 has no attachments — used by the+-- insert_attachments path test to add and then orphan-delete an .ics.++INSERT INTO email_links (id, macro_id, fusionauth_user_id, email_address, provider, is_sync_active, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-000000000b01', 'macro|[email protected]', '00000000-0000-0000-0000-000000000b01',+ '[email protected]', 'GMAIL', true, NOW(), NOW());++INSERT INTO email_contacts (id, link_id, email_address, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-0000000cb001',+ '00000000-0000-0000-0000-000000000b01',+ '[email protected]',+ NOW(), NOW());++INSERT INTO email_threads (id, link_id, inbox_visible, is_read, has_calendar_attachment, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-00000000b201',+ '00000000-0000-0000-0000-000000000b01',+ true, false, false, NOW(), NOW()),+ ('00000000-0000-0000-0000-00000000b202',+ '00000000-0000-0000-0000-000000000b01',+ true, false, true, NOW(), NOW()),+ ('00000000-0000-0000-0000-00000000b203',+ '00000000-0000-0000-0000-000000000b01',+ true, false, false, NOW(), NOW());++INSERT INTO email_messages (id, thread_id, link_id, provider_id, global_id, is_sent, from_contact_id, internal_date_ts,+ has_attachments, is_read, is_starred, is_draft, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-00000000b501',+ '00000000-0000-0000-0000-00000000b201',+ '00000000-0000-0000-0000-000000000b01',+ 'provider-msg-b501', 'gid-b501', FALSE,+ '00000000-0000-0000-0000-0000000cb001',+ '2025-01-05 10:00:00 +00:00',+ true, false, false, false, NOW(), NOW()),+ ('00000000-0000-0000-0000-00000000b502',+ '00000000-0000-0000-0000-00000000b201',+ '00000000-0000-0000-0000-000000000b01',+ 'provider-msg-b502', 'gid-b502', FALSE,+ '00000000-0000-0000-0000-0000000cb001',+ '2025-01-06 10:00:00 +00:00',+ false, false, false, false, NOW(), NOW()),+ ('00000000-0000-0000-0000-00000000b503',+ '00000000-0000-0000-0000-00000000b202',+ '00000000-0000-0000-0000-000000000b01',+ 'provider-msg-b503', 'gid-b503', FALSE,+ '00000000-0000-0000-0000-0000000cb001',+ '2025-01-07 10:00:00 +00:00',+ true, false, false, false, NOW(), NOW()),+ ('00000000-0000-0000-0000-00000000b504',+ '00000000-0000-0000-0000-00000000b203',+ '00000000-0000-0000-0000-000000000b01',+ 'provider-msg-b504', 'gid-b504', FALSE,+ '00000000-0000-0000-0000-0000000cb001',+ '2025-01-08 10:00:00 +00:00',+ false, false, false, false, NOW(), NOW());++INSERT INTO email_attachments (id, message_id, provider_attachment_id, filename, mime_type, size_bytes, content_id, created_at)+VALUES ('00000000-0000-0000-0000-0000001ba001',+ '00000000-0000-0000-0000-00000000b501',+ 'provider-att-b001', 'invite.ics', 'text/calendar', 1024, NULL, NOW()),+ ('00000000-0000-0000-0000-0000001ba002',+ '00000000-0000-0000-0000-00000000b503',+ 'provider-att-b002', 'notes.pdf', 'application/pdf', 2048, NULL, NOW());diff --git a/rust/cloud-storage/email_db_client/src/attachments/provider/mod.rs b/rust/cloud-storage/email_db_client/src/attachments/provider/mod.rsindex 22e28a2d05..56e4029ab1 100644--- a/rust/cloud-storage/email_db_client/src/attachments/provider/mod.rs+++ b/rust/cloud-storage/email_db_client/src/attachments/provider/mod.rs@@ -84,7 +84,17 @@ pub async fn insert_attachments(.await?;}+ // The thread-level flag only moves when a calendar attachment enters or+ // leaves this message's attachment set.+ let calendar_set_changed = orphaned_attachments+ .iter()+ .chain(new_attachments.iter())+ .any(|a| is_calendar_attachment(a.filename.as_deref(), a.mime_type.as_deref()));+if new_attachments.is_empty() {+ if calendar_set_changed {+ sync_thread_calendar_flag(&mut *tx, message_id).await?;+ }return Ok(());}@@ -136,6 +146,38 @@ pub async fn insert_attachments(.execute(&mut *tx).await?;+ if calendar_set_changed {+ sync_thread_calendar_flag(&mut *tx, message_id).await?;+ }++ Ok(())+}++/// Mirrors the SQL calendar-attachment predicate behind the CalendarOnly+/// email filter: `filename ILIKE '%.ics' OR mime_type IN ('text/calendar',+/// 'application/ics')`.+fn is_calendar_attachment(filename: Option<&str>, mime_type: Option<&str>) -> bool {+ filename.is_some_and(|f| f.to_ascii_lowercase().ends_with(".ics"))+ || mime_type.is_some_and(|m| m == "text/calendar" || m == "application/ics")+}++/// Resolves the message's thread and recomputes its denormalized+/// `has_calendar_attachment` flag.+async fn sync_thread_calendar_flag(+ tx: &mut sqlx::PgConnection,+ message_id: Uuid,+) -> anyhow::Result<()> {+ let thread_id: Option<Uuid> = sqlx::query_scalar!(+ "SELECT thread_id FROM email_messages WHERE id = $1",+ message_id+ )+ .fetch_optional(&mut *tx)+ .await?;++ if let Some(thread_id) = thread_id {+ crate::threads::update::sync_thread_calendar_flag(tx, thread_id).await?;+ }+Ok(())}diff --git a/rust/cloud-storage/email_db_client/src/attachments/provider/test.rs b/rust/cloud-storage/email_db_client/src/attachments/provider/test.rsindex 47a21b5ff6..77ecdf9a7e 100644--- a/rust/cloud-storage/email_db_client/src/attachments/provider/test.rs+++ b/rust/cloud-storage/email_db_client/src/attachments/provider/test.rs@@ -199,3 +199,63 @@ async fn fetch_db_attachments_in_bulk_returns_empty_for_nonexistent_messages(Ok(())}++async fn fetch_calendar_flag(pool: &Pool<Postgres>, thread_id: Uuid) -> Result<bool> {+ Ok(sqlx::query_scalar!(+ "SELECT has_calendar_attachment FROM email_threads WHERE id = $1",+ thread_id+ )+ .fetch_one(pool)+ .await?)+}++fn service_attachment(+ filename: &str,+ mime_type: &str,+ provider_id: &str,+) -> models_email::service::attachment::Attachment {+ models_email::service::attachment::Attachment {+ db_id: Uuid::new_v4(),+ provider_id: Some(provider_id.to_string()),+ data_url: None,+ filename: Some(filename.to_string()),+ mime_type: Some(mime_type.to_string()),+ size_bytes: Some(1024),+ sfs_id: None,+ content_id: None,+ }+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("sync_thread_calendar_flag"))+)]+async fn insert_attachments_sets_then_clears_calendar_flag(pool: Pool<Postgres>) -> Result<()> {+ let thread_id = Uuid::parse_str("00000000-0000-0000-0000-00000000b203")?;+ let message_id = Uuid::parse_str("00000000-0000-0000-0000-00000000b504")?;+ assert!(!fetch_calendar_flag(&pool, thread_id).await?);++ // Sync an .ics onto the message: the thread flag turns on.+ let mut tx = pool.begin().await?;+ let mut attachments = vec![service_attachment(+ "invite.ics",+ "text/calendar",+ "prov-ics-1",+ )];+ crate::attachments::provider::insert_attachments(&mut tx, message_id, &mut attachments).await?;+ tx.commit().await?;+ assert!(fetch_calendar_flag(&pool, thread_id).await?);++ // Re-sync with only a PDF: the .ics is orphan-deleted and the flag clears.+ let mut tx = pool.begin().await?;+ let mut attachments = vec![service_attachment(+ "notes.pdf",+ "application/pdf",+ "prov-pdf-1",+ )];+ crate::attachments::provider::insert_attachments(&mut tx, message_id, &mut attachments).await?;+ tx.commit().await?;+ assert!(!fetch_calendar_flag(&pool, thread_id).await?);++ Ok(())+}diff --git a/rust/cloud-storage/email_db_client/src/messages/delete.rs b/rust/cloud-storage/email_db_client/src/messages/delete.rsindex 733e553c2f..18618261e7 100644--- a/rust/cloud-storage/email_db_client/src/messages/delete.rs+++ b/rust/cloud-storage/email_db_client/src/messages/delete.rs@@ -24,6 +24,13 @@ pub async fn delete_message_with_tx(.await?;}+ // The message's attachments cascade with the delete, which may have+ // removed the thread's last calendar attachment. Drafts never have+ // email_attachments rows, so they can't move the flag.+ if !deleted_thread && !message.is_draft {+ threads::update::sync_thread_calendar_flag(&mut *tx, message.thread_db_id).await?;+ }+if deleted_thread {Ok(Some(message.thread_db_id))} else {diff --git a/rust/cloud-storage/email_db_client/src/messages/test.rs b/rust/cloud-storage/email_db_client/src/messages/test.rsindex 1d10b24238..d5e0877679 100644--- a/rust/cloud-storage/email_db_client/src/messages/test.rs+++ b/rust/cloud-storage/email_db_client/src/messages/test.rs@@ -296,3 +296,49 @@ async fn get_scheduled_db_messages_returns_empty_for_nonexistent_link(Ok(())}++async fn fetch_calendar_flag(pool: &Pool<Postgres>, thread_id: Uuid) -> anyhow::Result<bool> {+ Ok(sqlx::query_scalar!(+ "SELECT has_calendar_attachment FROM email_threads WHERE id = $1",+ thread_id+ )+ .fetch_one(pool)+ .await?)+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../fixtures", scripts("sync_thread_calendar_flag"))+)]+async fn delete_message_clears_calendar_flag_when_last_ics_message_removed(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let thread_id = Uuid::parse_str("00000000-0000-0000-0000-00000000b201")?;+ let ics_message_id = Uuid::parse_str("00000000-0000-0000-0000-00000000b501")?;+ let fusionauth_user_id = "00000000-0000-0000-0000-000000000b01";++ // Establish the flag from the fixture's .ics attachment.+ let mut conn = pool.acquire().await?;+ crate::threads::update::sync_thread_calendar_flag(&mut conn, thread_id).await?;+ drop(conn);+ assert!(fetch_calendar_flag(&pool, thread_id).await?);++ let message = crate::messages::get_simple_messages::get_simple_message(+ &pool,+ &ics_message_id,+ fusionauth_user_id,+ )+ .await?+ .expect("fixture message present");++ let mut tx = pool.begin().await?;+ let deleted_thread =+ crate::messages::delete::delete_message_with_tx(&mut tx, &message, true).await?;+ tx.commit().await?;++ // The thread survives (a second message remains) and the flag flips off+ // because its attachments cascaded away with the message.+ assert!(deleted_thread.is_none());+ assert!(!fetch_calendar_flag(&pool, thread_id).await?);+ Ok(())+}diff --git a/rust/cloud-storage/email_db_client/src/threads/update.rs b/rust/cloud-storage/email_db_client/src/threads/update.rsindex 2a1348df46..09f23869b3 100644--- a/rust/cloud-storage/email_db_client/src/threads/update.rs+++ b/rust/cloud-storage/email_db_client/src/threads/update.rs@@ -1,3 +1,6 @@+#[cfg(test)]+mod test;+use crate::messages::get::fetch_messages_metadata;use chrono::{DateTime, Utc};@@ -133,6 +136,40 @@ pub async fn update_thread_provider_id(Ok(())}+/// Recomputes the denormalized `email_threads.has_calendar_attachment` flag+/// from the thread's current attachment set. Mirrors the CalendarOnly+/// predicate in the email crate's dynamic query builder.+#[tracing::instrument(skip(tx), err)]+pub async fn sync_thread_calendar_flag(+ tx: &mut sqlx::PgConnection,+ thread_db_id: Uuid,+) -> anyhow::Result<()> {+ sqlx::query!(+ r#"+ UPDATE email_threads t+ SET has_calendar_attachment = calc.has_cal+ FROM (+ SELECT EXISTS (+ SELECT 1+ FROM email_messages m+ JOIN email_attachments a ON a.message_id = m.id+ WHERE m.thread_id = $1+ AND (a.filename ILIKE '%.ics'+ OR a.mime_type = 'text/calendar'+ OR a.mime_type = 'application/ics')+ ) AS has_cal+ ) calc+ WHERE t.id = $1+ AND t.has_calendar_attachment IS DISTINCT FROM calc.has_cal+ "#,+ thread_db_id+ )+ .execute(tx)+ .await?;++ Ok(())+}+// Updates a thread's metadata (archived status, latest_timestamps)#[tracing::instrument(skip(tx), err)]pub async fn update_thread_metadata(diff --git a/rust/cloud-storage/email_db_client/src/threads/update/test.rs b/rust/cloud-storage/email_db_client/src/threads/update/test.rsnew file mode 100644index 0000000000..7292f5a3da--- /dev/null+++ b/rust/cloud-storage/email_db_client/src/threads/update/test.rs@@ -0,0 +1,64 @@+use super::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::{Pool, Postgres};++const THREAD_WITH_ICS: &str = "00000000-0000-0000-0000-00000000b201";+const THREAD_STALE_TRUE_PDF_ONLY: &str = "00000000-0000-0000-0000-00000000b202";++async fn fetch_flag(pool: &Pool<Postgres>, thread_id: &str) -> anyhow::Result<bool> {+ Ok(sqlx::query_scalar!(+ "SELECT has_calendar_attachment FROM email_threads WHERE id = $1",+ Uuid::parse_str(thread_id)?+ )+ .fetch_one(pool)+ .await?)+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("sync_thread_calendar_flag"))+)]+async fn sync_sets_flag_when_thread_has_calendar_attachment(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ assert!(!fetch_flag(&pool, THREAD_WITH_ICS).await?);++ let mut conn = pool.acquire().await?;+ sync_thread_calendar_flag(&mut conn, Uuid::parse_str(THREAD_WITH_ICS)?).await?;++ assert!(fetch_flag(&pool, THREAD_WITH_ICS).await?);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("sync_thread_calendar_flag"))+)]+async fn sync_clears_stale_flag_when_no_calendar_attachment(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ // Fixture marks this thread true even though its only attachment is a PDF.+ assert!(fetch_flag(&pool, THREAD_STALE_TRUE_PDF_ONLY).await?);++ let mut conn = pool.acquire().await?;+ sync_thread_calendar_flag(&mut conn, Uuid::parse_str(THREAD_STALE_TRUE_PDF_ONLY)?).await?;++ assert!(!fetch_flag(&pool, THREAD_STALE_TRUE_PDF_ONLY).await?);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("sync_thread_calendar_flag"))+)]+async fn sync_is_stable_when_flag_already_matches(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let thread_id = Uuid::parse_str(THREAD_WITH_ICS)?;+ let mut conn = pool.acquire().await?;+ sync_thread_calendar_flag(&mut conn, thread_id).await?;+ assert!(fetch_flag(&pool, THREAD_WITH_ICS).await?);++ // Re-sync with no attachment changes: the value must stay put.+ sync_thread_calendar_flag(&mut conn, thread_id).await?;+ assert!(fetch_flag(&pool, THREAD_WITH_ICS).await?);+ Ok(())+}diff --git a/rust/cloud-storage/email_service/Cargo.toml b/rust/cloud-storage/email_service/Cargo.tomlindex 6481d89304..d3c70ef435 100644--- a/rust/cloud-storage/email_service/Cargo.toml+++ b/rust/cloud-storage/email_service/Cargo.toml@@ -13,6 +13,10 @@ path = "src/main.rs"name = "backfill_contacts"path = "src/bin/backfill_contacts/backfill_contacts.rs"+[[bin]]+name = "backfill_calendar_flags"+path = "src/bin/backfill_calendar_flags/backfill_calendar_flags.rs"+[[bin]]name = "backfill_gmail_attachments"path = "src/bin/backfill_gmail_attachments/backfill_gmail_attachments.rs"diff --git a/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/backfill_calendar_flags.rs b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/backfill_calendar_flags.rsnew file mode 100644index 0000000000..85b9decd2c--- /dev/null+++ b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/backfill_calendar_flags.rs@@ -0,0 +1,102 @@+//! # Calendar Flag Backfill Utility+//!+//! Backfills the denormalized `email_threads.has_calendar_attachment` column:+//! for each user, flags every thread that has at least one iCalendar+//! attachment (`*.ics` filename or `text/calendar` / `application/ics` mime).+//! Only ever sets the flag to true — steady-state maintenance (including+//! clearing it on deletes) is handled by the ingest and delete paths in+//! `email_db_client`.+//!+//! ## Required Environment Variables:+//! - `DATABASE_URL`: The connection string for the PostgreSQL database.+//!+//! ## Optional Environment Variables:+//! - `MACRO_IDS`: Comma-separated macro IDs to backfill. When unset, every+//! user with an email link is processed.+//! - `CONCURRENCY`: Number of users processed concurrently (defaults to 10).++mod config;+mod process;++use anyhow::Context;+use futures::stream::{self, StreamExt};+use macro_entrypoint::MacroEntrypoint;+use sqlx::postgres::PgPoolOptions;++#[tokio::main]+async fn main() -> anyhow::Result<()> {+ println!("Loading configuration...");+ MacroEntrypoint::default().init();+ let config = config::Config::from_env().context("Failed to load configuration")?;++ println!("Connecting to the database...");+ let db_pool = PgPoolOptions::new()+ .min_connections(1)+ .max_connections(config.concurrency as u32)+ .connect(&config.database_url)+ .await+ .context("Could not connect to db")?;++ let macro_ids: Vec<String> = match &config.macro_ids {+ Some(ids) => ids.split(',').map(|id| id.trim().to_string()).collect(),+ None => process::fetch_all_macro_ids(&db_pool)+ .await+ .context("Failed to fetch macro IDs")?,+ };++ let total_users = macro_ids.len();+ println!(+ "Processing {total_users} macro IDs with concurrency {}",+ config.concurrency+ );++ let results: Vec<(String, anyhow::Result<u64>)> = stream::iter(macro_ids)+ .map(|macro_id| {+ let db_pool = db_pool.clone();+ async move {+ let result = process::process_macro_id(&db_pool, ¯o_id).await;+ (macro_id, result)+ }+ })+ .buffer_unordered(config.concurrency)+ .enumerate()+ .map(|(index, (macro_id, result))| {+ match &result {+ Ok(flagged) => println!(+ "=== Completed {macro_id} ({}/{total_users}): flagged {flagged} threads ===",+ index + 1+ ),+ Err(e) => println!(+ "=== Failed {macro_id} ({}/{total_users}): {e:?} ===",+ index + 1+ ),+ }+ (macro_id, result)+ })+ .collect()+ .await;++ let total_flagged: u64 = results.iter().filter_map(|(_, r)| r.as_ref().ok()).sum();+ let failures: Vec<&String> = results+ .iter()+ .filter(|(_, r)| r.is_err())+ .map(|(id, _)| id)+ .collect();++ println!(+ "\n=== All macro IDs processed: {total_flagged} threads flagged, {} failures ===",+ failures.len()+ );+ if !failures.is_empty() {+ anyhow::bail!(+ "{} macro IDs failed ({}); rerun to retry (already-flagged threads are skipped)",+ failures.len(),+ failures+ .iter()+ .map(|s| s.as_str())+ .collect::<Vec<_>>()+ .join(", ")+ );+ }+ Ok(())+}diff --git a/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/config.rs b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/config.rsnew file mode 100644index 0000000000..fa099457f7--- /dev/null+++ b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/config.rs@@ -0,0 +1,41 @@+use anyhow::Context;+use macro_env_var::env_vars;++/// Holds all configuration loaded from environment variables.+pub struct Config {+ /// Comma-separated macro IDs to backfill. When unset, every user with an+ /// email link is processed.+ pub macro_ids: Option<String>,+ pub database_url: String,+ /// Number of users processed concurrently. Defaults to 10, capped at 50+ /// to bound DB connection usage.+ pub concurrency: usize,+}++env_vars! {+ struct MacroIds;+ struct DatabaseUrl;+ struct Concurrency;+}++impl Config {+ /// Creates a new `Config` instance by reading from environment variables.+ /// Returns an error if any required variable is not set.+ pub fn from_env() -> anyhow::Result<Self> {+ let concurrency = match Concurrency::new().ok() {+ Some(v) => v+ .parse::<usize>()+ .context("CONCURRENCY is not a number")?+ .clamp(1, 50),+ None => 10,+ };++ Ok(Self {+ macro_ids: MacroIds::new().ok().map(|v| v.to_string()),+ database_url: DatabaseUrl::new()+ .context("DATABASE_URL not set")?+ .to_string(),+ concurrency,+ })+ }+}diff --git a/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/process.rs b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/process.rsnew file mode 100644index 0000000000..b36c20d472--- /dev/null+++ b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/process.rs@@ -0,0 +1,60 @@+use sqlx::types::Uuid;++/// Flags every thread on one of the user's links that has at least one+/// calendar attachment. One statement per link keeps transactions small and+/// the progress output per-mailbox. Returns the number of threads flagged.+pub async fn process_macro_id(pool: &sqlx::PgPool, macro_id: &str) -> anyhow::Result<u64> {+ let link_ids: Vec<Uuid> =+ sqlx::query_scalar!("SELECT id FROM email_links WHERE macro_id = $1", macro_id)+ .fetch_all(pool)+ .await?;++ if link_ids.is_empty() {+ println!("No email links found for {macro_id}.");+ return Ok(0);+ }++ let mut total_flagged = 0u64;+ for link_id in link_ids {+ // Mirrors the CalendarOnly predicate in the email crate's dynamic+ // query builder and email_db_client::threads::update::sync_thread_calendar_flag.+ let flagged = sqlx::query!(+ r#"+ UPDATE email_threads t+ SET has_calendar_attachment = true+ FROM (+ SELECT DISTINCT m.thread_id+ FROM email_messages m+ JOIN email_attachments a ON a.message_id = m.id+ WHERE m.link_id = $1+ AND (a.filename ILIKE '%.ics'+ OR a.mime_type = 'text/calendar'+ OR a.mime_type = 'application/ics')+ ) cal+ WHERE t.id = cal.thread_id+ AND NOT t.has_calendar_attachment+ "#,+ link_id+ )+ .execute(pool)+ .await?+ .rows_affected();++ // Prefix with the user so interleaved concurrent output stays readable.+ println!("[{macro_id}] link {link_id}: flagged {flagged} threads");+ total_flagged += flagged;+ }++ Ok(total_flagged)+}++/// Every macro ID that owns at least one email link. Connected secondary+/// mailboxes carry their own macro_id row in email_links, so iterating these+/// covers every link exactly once.+pub async fn fetch_all_macro_ids(pool: &sqlx::PgPool) -> anyhow::Result<Vec<String>> {+ Ok(+ sqlx::query_scalar!("SELECT DISTINCT macro_id FROM email_links ORDER BY macro_id")+ .fetch_all(pool)+ .await?,+ )+}diff --git a/rust/cloud-storage/macro_db_client/migrations/20260706155906_add_email_threads_has_calendar_attachment.sql b/rust/cloud-storage/macro_db_client/migrations/20260706155906_add_email_threads_has_calendar_attachment.sqlnew file mode 100644index 0000000000..e157907d7d--- /dev/null+++ b/rust/cloud-storage/macro_db_client/migrations/20260706155906_add_email_threads_has_calendar_attachment.sql@@ -0,0 +1,13 @@+-- Denormalized "thread has a calendar attachment" flag for the CalendarOnly+-- soup filter. Deriving it at query time joins email_attachments against+-- email_messages (hundreds of thousands of rows per link) on every page load.+-- Maintained at attachment ingest and message delete (email_db_client);+-- existing threads are flagged by the backfill_calendar_flags util in+-- email_service.+ALTER TABLE email_threads+ ADD COLUMN IF NOT EXISTS has_calendar_attachment boolean NOT NULL DEFAULT false;++-- Superseded by idx_email_threads_calendar_view (next migration), which adds+-- the sort key. CREATE INDEX CONCURRENTLY can't run in a transaction, so it+-- lives in its own no-transaction migration.+DROP INDEX IF EXISTS idx_email_threads_calendar_link_id;diff --git a/rust/cloud-storage/macro_db_client/migrations/20260706175458_email_threads_calendar_view_index.sql b/rust/cloud-storage/macro_db_client/migrations/20260706175458_email_threads_calendar_view_index.sqlnew file mode 100644index 0000000000..2e7eab2df9--- /dev/null+++ b/rust/cloud-storage/macro_db_client/migrations/20260706175458_email_threads_calendar_view_index.sql@@ -0,0 +1,8 @@+-- no-transaction+-- Matches the CalendarOnly candidate scan's ORDER BY+-- (COALESCE(latest_non_spam_message_ts, updated_at) DESC, id DESC) so+-- per-link ordered scans stop at the LIMIT instead of fetching and sorting+-- every flagged thread.+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_email_threads_calendar_view+ ON email_threads (link_id, (COALESCE(latest_non_spam_message_ts, updated_at)) DESC, id DESC)+ WHERE has_calendar_attachment;