Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(email): denormalize thread calendar-attachment flag (#4515)

Evan Hutnik44daee41d535
diff --git a/rust/cloud-storage/.sqlx/query-2810a4a2c8ea8504b36826226c558f1bde9259ab8a5dd1fd9556e43e2fa6513e.json b/rust/cloud-storage/.sqlx/query-2810a4a2c8ea8504b36826226c558f1bde9259ab8a5dd1fd9556e43e2fa6513e.json
new file mode 100644
index 0000000000..0defaa5187
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-2810a4a2c8ea8504b36826226c558f1bde9259ab8a5dd1fd9556e43e2fa6513e.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE email_threads t\n SET has_calendar_attachment = calc.has_cal\n FROM (\n SELECT EXISTS (\n SELECT 1\n FROM email_messages m\n JOIN email_attachments a ON a.message_id = m.id\n WHERE m.thread_id = $1\n AND (a.filename ILIKE '%.ics'\n OR a.mime_type = 'text/calendar'\n OR a.mime_type = 'application/ics')\n ) AS has_cal\n ) calc\n WHERE t.id = $1\n AND t.has_calendar_attachment IS DISTINCT FROM calc.has_cal\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "2810a4a2c8ea8504b36826226c558f1bde9259ab8a5dd1fd9556e43e2fa6513e"
+}
diff --git a/rust/cloud-storage/.sqlx/query-76ee78dac93f9184df7cd719f7b2b52709675e92466ec5907f1ded5bac7b21f2.json b/rust/cloud-storage/.sqlx/query-76ee78dac93f9184df7cd719f7b2b52709675e92466ec5907f1ded5bac7b21f2.json
new file mode 100644
index 0000000000..073e543bbd
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-76ee78dac93f9184df7cd719f7b2b52709675e92466ec5907f1ded5bac7b21f2.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE email_threads t\n SET has_calendar_attachment = true\n FROM (\n SELECT DISTINCT m.thread_id\n FROM email_messages m\n JOIN email_attachments a ON a.message_id = m.id\n WHERE m.link_id = $1\n AND (a.filename ILIKE '%.ics'\n OR a.mime_type = 'text/calendar'\n OR a.mime_type = 'application/ics')\n ) cal\n WHERE t.id = cal.thread_id\n AND NOT t.has_calendar_attachment\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "76ee78dac93f9184df7cd719f7b2b52709675e92466ec5907f1ded5bac7b21f2"
+}
diff --git a/rust/cloud-storage/.sqlx/query-855755b529207245d323d939a9d1718c12a6509c805f59bff9997e661f47142f.json b/rust/cloud-storage/.sqlx/query-855755b529207245d323d939a9d1718c12a6509c805f59bff9997e661f47142f.json
new file mode 100644
index 0000000000..64f2b0796a
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-855755b529207245d323d939a9d1718c12a6509c805f59bff9997e661f47142f.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT thread_id FROM email_messages WHERE id = $1",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "thread_id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "855755b529207245d323d939a9d1718c12a6509c805f59bff9997e661f47142f"
+}
diff --git a/rust/cloud-storage/.sqlx/query-b0aa946e4aba01ed02a9f633f563437e88be9f676e95ec13725bc598614bf24c.json b/rust/cloud-storage/.sqlx/query-b0aa946e4aba01ed02a9f633f563437e88be9f676e95ec13725bc598614bf24c.json
new file mode 100644
index 0000000000..3ff7622ee4
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-b0aa946e4aba01ed02a9f633f563437e88be9f676e95ec13725bc598614bf24c.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT id FROM email_links WHERE macro_id = $1",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "b0aa946e4aba01ed02a9f633f563437e88be9f676e95ec13725bc598614bf24c"
+}
diff --git a/rust/cloud-storage/.sqlx/query-b84715ccb1c38d74d76b11969b595a59db45d27a49f994583c9619b1a1df4fa3.json b/rust/cloud-storage/.sqlx/query-b84715ccb1c38d74d76b11969b595a59db45d27a49f994583c9619b1a1df4fa3.json
new file mode 100644
index 0000000000..144d55d4c0
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-b84715ccb1c38d74d76b11969b595a59db45d27a49f994583c9619b1a1df4fa3.json
@@ -0,0 +1,20 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT DISTINCT macro_id FROM email_links ORDER BY macro_id",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "macro_id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "b84715ccb1c38d74d76b11969b595a59db45d27a49f994583c9619b1a1df4fa3"
+}
diff --git a/rust/cloud-storage/.sqlx/query-e802b95145183d9bec6fb5f542fe93d392fdcf0fbebc9935f263f3f4fd1d8c34.json b/rust/cloud-storage/.sqlx/query-e802b95145183d9bec6fb5f542fe93d392fdcf0fbebc9935f263f3f4fd1d8c34.json
deleted file mode 100644
index e30af0774f..0000000000
--- a/rust/cloud-storage/.sqlx/query-e802b95145183d9bec6fb5f542fe93d392fdcf0fbebc9935f263f3f4fd1d8c34.json
+++ /dev/null
@@ -1,128 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT\n t.id,\n t.provider_id,\n t.inbox_visible,\n t.is_read,\n t.effective_ts AS \"sort_ts!\",\n t.created_at AS \"created_at!\",\n t.updated_at AS \"updated_at!\",\n t.project_id,\n t.viewed_at AS \"viewed_at?\",\n lmp.subject AS \"name?\",\n lmp.snippet AS \"snippet?\",\n lmp.is_draft,\n (\n SELECT EXISTS (\n SELECT 1\n FROM email_messages m_imp\n JOIN email_message_labels ml ON m_imp.id = ml.message_id\n JOIN email_labels l ON ml.label_id = l.id\n WHERE m_imp.thread_id = t.id\n AND l.name = 'IMPORTANT'\n AND l.link_id = t.link_id\n )\n ) AS \"is_important!\",\n c.email_address AS \"sender_email?\",\n COALESCE(lmp.from_name, c.name) AS \"sender_name?\",\n c.sfs_photo_url as \"sender_photo_url?\",\n el.macro_id AS \"owner_id!\",\n el.id AS \"link_id!\"\n FROM (\n -- Step 1: Efficiently find and sort ONLY the top N+1 candidate threads.\n -- This subquery only touches `threads` and `user_history`.\n SELECT\n t.id,\n t.provider_id,\n t.link_id,\n t.inbox_visible,\n t.is_read,\n t.project_id,\n t.latest_outbound_message_ts AS created_at,\n t.latest_outbound_message_ts AS updated_at,\n uh.updated_at AS viewed_at,\n CASE $5 -- sort_method_str\n WHEN 'viewed_at' THEN COALESCE(uh.\"updated_at\", '1970-01-01 00:00:00+00')\n WHEN 'viewed_updated' THEN COALESCE(uh.updated_at, t.latest_outbound_message_ts)\n ELSE t.latest_outbound_message_ts\n END AS effective_ts\n FROM email_threads t\n LEFT JOIN email_user_history uh ON uh.thread_id = t.id AND uh.link_id = t.link_id\n WHERE\n t.link_id = ANY($1)\n AND t.latest_outbound_message_ts IS NOT NULL\n \n -- Cursor logic moved inside for maximum efficiency\n AND (($3::timestamptz IS NULL) OR (\n -- This CASE must exactly match the one that defines `effective_ts`\n CASE $5 -- sort_method_str\n WHEN 'viewed_at' THEN COALESCE(uh.\"updated_at\", '1970-01-01 00:00:00+00')\n WHEN 'viewed_updated' THEN COALESCE(uh.updated_at, t.latest_outbound_message_ts)\n ELSE t.latest_outbound_message_ts\n END, t.id\n ) < ($3::timestamptz, $4::uuid))\n ORDER BY effective_ts DESC, t.updated_at DESC\n LIMIT $2\n ) AS t\n -- Step 2: For EACH of the limited threads from above, find its latest SENT, non-trashed message.\n CROSS JOIN LATERAL (\n SELECT\n m.subject,\n m.snippet,\n m.from_contact_id,\n m.from_name,\n m.is_draft\n FROM email_messages m\n WHERE m.thread_id = t.id\n AND m.is_sent = TRUE -- This condition is specific to the \"Sent\" view\n AND NOT EXISTS (\n SELECT 1 FROM email_message_labels ml JOIN email_labels l ON ml.label_id = l.id\n WHERE ml.message_id = m.id AND l.name = 'TRASH' AND l.link_id = t.link_id\n )\n ORDER BY m.internal_date_ts DESC\n LIMIT 1\n ) AS lmp\n -- Step 3: Join to get the sender's details for the final result set.\n LEFT JOIN email_contacts c ON lmp.from_contact_id = c.id\n JOIN email_links el ON t.link_id = el.id\n -- Final ordering is preserved because the input `t` is already sorted.\n ORDER BY t.effective_ts DESC, t.updated_at DESC\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 1,
- "name": "provider_id",
- "type_info": "Text"
- },
- {
- "ordinal": 2,
- "name": "inbox_visible",
- "type_info": "Bool"
- },
- {
- "ordinal": 3,
- "name": "is_read",
- "type_info": "Bool"
- },
- {
- "ordinal": 4,
- "name": "sort_ts!",
- "type_info": "Timestamptz"
- },
- {
- "ordinal": 5,
- "name": "created_at!",
- "type_info": "Timestamptz"
- },
- {
- "ordinal": 6,
- "name": "updated_at!",
- "type_info": "Timestamptz"
- },
- {
- "ordinal": 7,
- "name": "project_id",
- "type_info": "Text"
- },
- {
- "ordinal": 8,
- "name": "viewed_at?",
- "type_info": "Timestamptz"
- },
- {
- "ordinal": 9,
- "name": "name?",
- "type_info": "Text"
- },
- {
- "ordinal": 10,
- "name": "snippet?",
- "type_info": "Text"
- },
- {
- "ordinal": 11,
- "name": "is_draft",
- "type_info": "Bool"
- },
- {
- "ordinal": 12,
- "name": "is_important!",
- "type_info": "Bool"
- },
- {
- "ordinal": 13,
- "name": "sender_email?",
- "type_info": "Varchar"
- },
- {
- "ordinal": 14,
- "name": "sender_name?",
- "type_info": "Varchar"
- },
- {
- "ordinal": 15,
- "name": "sender_photo_url?",
- "type_info": "Text"
- },
- {
- "ordinal": 16,
- "name": "owner_id!",
- "type_info": "Text"
- },
- {
- "ordinal": 17,
- "name": "link_id!",
- "type_info": "Uuid"
- }
- ],
- "parameters": {
- "Left": [
- "UuidArray",
- "Int8",
- "Timestamptz",
- "Uuid",
- "Text"
- ]
- },
- "nullable": [
- false,
- true,
- false,
- false,
- null,
- true,
- true,
- true,
- false,
- true,
- true,
- false,
- null,
- false,
- null,
- true,
- false,
- false
- ]
- },
- "hash": "e802b95145183d9bec6fb5f542fe93d392fdcf0fbebc9935f263f3f4fd1d8c34"
-}
diff --git a/rust/cloud-storage/email/fixtures/email_dynamic_query_calendar.sql b/rust/cloud-storage/email/fixtures/email_dynamic_query_calendar.sql
index 002c766c14..ee69181325 100644
--- a/rust/cloud-storage/email/fixtures/email_dynamic_query_calendar.sql
+++ b/rust/cloud-storage/email/fixtures/email_dynamic_query_calendar.sql
@@ -33,3 +33,16 @@ VALUES
-- Thread 7 (inbox with CC): only a non-calendar attachment → should NOT match
('90000007-0000-0000-0000-000000000007', '30000007-0000-0000-0000-000000000007',
'report.pdf', 'application/pdf', NOW());
+
+-- Maintain the denormalized flag the CalendarOnly filter reads (set at
+-- attachment ingest in production; mirrored here for the raw inserts above).
+UPDATE email_threads t
+SET has_calendar_attachment = true
+WHERE t.id IN (
+ SELECT m.thread_id
+ FROM email_attachments a
+ JOIN email_messages m ON m.id = a.message_id
+ WHERE a.filename ILIKE '%.ics'
+ OR a.mime_type = 'text/calendar'
+ OR a.mime_type = 'application/ics'
+);
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs
index bed2d12f3e..4bcebb70c3 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs
@@ -761,19 +761,11 @@ pub(super) fn build_thread_email_filter(
f
}
- filter_ast::ExprFrame::Literal(EmailLiteral::CalendarOnly(true)) => SqlFragment::raw(
- r#"EXISTS (
- SELECT 1
- FROM email_messages m_cal
- JOIN email_attachments a_cal ON a_cal.message_id = m_cal.id
- WHERE m_cal.thread_id = t.id
- AND (
- a_cal.filename ILIKE '%.ics'
- OR a_cal.mime_type = 'text/calendar'
- OR a_cal.mime_type = 'application/ics'
- )
- )"#,
- ),
+ // Denormalized flag maintained at attachment ingest — deriving this
+ // from email_attachments at query time is prohibitively slow.
+ filter_ast::ExprFrame::Literal(EmailLiteral::CalendarOnly(true)) => {
+ SqlFragment::raw("t.has_calendar_attachment")
+ }
filter_ast::ExprFrame::Literal(EmailLiteral::CalendarOnly(false)) => {
SqlFragment::raw("TRUE")
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs
index da2c309b43..92f7684821 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs
@@ -1229,17 +1229,34 @@ fn test_full_query_emits_matching_threads_cte_and_in_reference() {
}
#[test]
-fn test_build_thread_email_filter_calendar_only_true_emits_ics_exists() {
+fn test_build_thread_email_filter_calendar_only_true_uses_thread_flag() {
let expr = Expr::Literal(EmailLiteral::CalendarOnly(true));
let result = build_thread_email_filter(&expr, DEFAULT_SORT_TS);
let debug = result.to_debug_sql();
- assert!(debug.contains("EXISTS"));
- assert!(debug.contains("email_attachments"));
- assert!(debug.contains("m_cal.thread_id = t.id"));
- assert!(debug.contains("a_cal.filename ILIKE '%.ics'"));
- assert!(debug.contains("a_cal.mime_type = 'text/calendar'"));
- assert!(debug.contains("a_cal.mime_type = 'application/ics'"));
+ assert!(debug.contains("t.has_calendar_attachment"));
+ // No per-thread attachment probe remains in the candidate WHERE.
+ assert!(!debug.contains("email_attachments"));
+}
+
+#[test]
+fn test_full_query_calendar_only_uses_thread_flag_without_cte() {
+ let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);
+ let expr = Expr::Literal(EmailLiteral::CalendarOnly(true));
+ let sql = super::query::debug_build_query_sql(&view, &expr);
+
+ assert!(
+ sql.contains("t.has_calendar_attachment"),
+ "candidate WHERE must use the denormalized flag: {sql}"
+ );
+ assert!(
+ !sql.contains("calendar_threads"),
+ "calendar CTE should be gone: {sql}"
+ );
+ assert!(
+ !sql.contains("email_attachments"),
+ "no attachment scan should remain: {sql}"
+ );
}
#[test]
diff --git a/rust/cloud-storage/email_db_client/fixtures/sync_thread_calendar_flag.sql b/rust/cloud-storage/email_db_client/fixtures/sync_thread_calendar_flag.sql
new file mode 100644
index 0000000000..7af259e183
--- /dev/null
+++ b/rust/cloud-storage/email_db_client/fixtures/sync_thread_calendar_flag.sql
@@ -0,0 +1,68 @@
+-- Fixture for the denormalized email_threads.has_calendar_attachment flag.
+--
+-- Thread 1 (flag=false): message b501 has an .ics attachment, message b502
+-- has none — sync should set the flag true; deleting b501 should clear it.
+-- Thread 2 (flag=true, stale on purpose): message b503 has only a PDF —
+-- sync should clear the flag.
+-- Thread 3 (flag=false): message b504 has no attachments — used by the
+-- insert_attachments path test to add and then orphan-delete an .ics.
+
+INSERT INTO email_links (id, macro_id, fusionauth_user_id, email_address, provider, is_sync_active, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-000000000b01', 'macro|[email protected]', '00000000-0000-0000-0000-000000000b01',
+ '[email protected]', 'GMAIL', true, NOW(), NOW());
+
+INSERT INTO email_contacts (id, link_id, email_address, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-0000000cb001',
+ '00000000-0000-0000-0000-000000000b01',
+ NOW(), NOW());
+
+INSERT INTO email_threads (id, link_id, inbox_visible, is_read, has_calendar_attachment, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-00000000b201',
+ '00000000-0000-0000-0000-000000000b01',
+ true, false, false, NOW(), NOW()),
+ ('00000000-0000-0000-0000-00000000b202',
+ '00000000-0000-0000-0000-000000000b01',
+ true, false, true, NOW(), NOW()),
+ ('00000000-0000-0000-0000-00000000b203',
+ '00000000-0000-0000-0000-000000000b01',
+ true, false, false, NOW(), NOW());
+
+INSERT INTO email_messages (id, thread_id, link_id, provider_id, global_id, is_sent, from_contact_id, internal_date_ts,
+ has_attachments, is_read, is_starred, is_draft, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-00000000b501',
+ '00000000-0000-0000-0000-00000000b201',
+ '00000000-0000-0000-0000-000000000b01',
+ 'provider-msg-b501', 'gid-b501', FALSE,
+ '00000000-0000-0000-0000-0000000cb001',
+ '2025-01-05 10:00:00 +00:00',
+ true, false, false, false, NOW(), NOW()),
+ ('00000000-0000-0000-0000-00000000b502',
+ '00000000-0000-0000-0000-00000000b201',
+ '00000000-0000-0000-0000-000000000b01',
+ 'provider-msg-b502', 'gid-b502', FALSE,
+ '00000000-0000-0000-0000-0000000cb001',
+ '2025-01-06 10:00:00 +00:00',
+ false, false, false, false, NOW(), NOW()),
+ ('00000000-0000-0000-0000-00000000b503',
+ '00000000-0000-0000-0000-00000000b202',
+ '00000000-0000-0000-0000-000000000b01',
+ 'provider-msg-b503', 'gid-b503', FALSE,
+ '00000000-0000-0000-0000-0000000cb001',
+ '2025-01-07 10:00:00 +00:00',
+ true, false, false, false, NOW(), NOW()),
+ ('00000000-0000-0000-0000-00000000b504',
+ '00000000-0000-0000-0000-00000000b203',
+ '00000000-0000-0000-0000-000000000b01',
+ 'provider-msg-b504', 'gid-b504', FALSE,
+ '00000000-0000-0000-0000-0000000cb001',
+ '2025-01-08 10:00:00 +00:00',
+ false, false, false, false, NOW(), NOW());
+
+INSERT INTO email_attachments (id, message_id, provider_attachment_id, filename, mime_type, size_bytes, content_id, created_at)
+VALUES ('00000000-0000-0000-0000-0000001ba001',
+ '00000000-0000-0000-0000-00000000b501',
+ 'provider-att-b001', 'invite.ics', 'text/calendar', 1024, NULL, NOW()),
+ ('00000000-0000-0000-0000-0000001ba002',
+ '00000000-0000-0000-0000-00000000b503',
+ 'provider-att-b002', 'notes.pdf', 'application/pdf', 2048, NULL, NOW());
diff --git a/rust/cloud-storage/email_db_client/src/attachments/provider/mod.rs b/rust/cloud-storage/email_db_client/src/attachments/provider/mod.rs
index 22e28a2d05..56e4029ab1 100644
--- a/rust/cloud-storage/email_db_client/src/attachments/provider/mod.rs
+++ b/rust/cloud-storage/email_db_client/src/attachments/provider/mod.rs
@@ -84,7 +84,17 @@ pub async fn insert_attachments(
.await?;
}
+ // The thread-level flag only moves when a calendar attachment enters or
+ // leaves this message's attachment set.
+ let calendar_set_changed = orphaned_attachments
+ .iter()
+ .chain(new_attachments.iter())
+ .any(|a| is_calendar_attachment(a.filename.as_deref(), a.mime_type.as_deref()));
+
if new_attachments.is_empty() {
+ if calendar_set_changed {
+ sync_thread_calendar_flag(&mut *tx, message_id).await?;
+ }
return Ok(());
}
@@ -136,6 +146,38 @@ pub async fn insert_attachments(
.execute(&mut *tx)
.await?;
+ if calendar_set_changed {
+ sync_thread_calendar_flag(&mut *tx, message_id).await?;
+ }
+
+ Ok(())
+}
+
+/// Mirrors the SQL calendar-attachment predicate behind the CalendarOnly
+/// email filter: `filename ILIKE '%.ics' OR mime_type IN ('text/calendar',
+/// 'application/ics')`.
+fn is_calendar_attachment(filename: Option<&str>, mime_type: Option<&str>) -> bool {
+ filename.is_some_and(|f| f.to_ascii_lowercase().ends_with(".ics"))
+ || mime_type.is_some_and(|m| m == "text/calendar" || m == "application/ics")
+}
+
+/// Resolves the message's thread and recomputes its denormalized
+/// `has_calendar_attachment` flag.
+async fn sync_thread_calendar_flag(
+ tx: &mut sqlx::PgConnection,
+ message_id: Uuid,
+) -> anyhow::Result<()> {
+ let thread_id: Option<Uuid> = sqlx::query_scalar!(
+ "SELECT thread_id FROM email_messages WHERE id = $1",
+ message_id
+ )
+ .fetch_optional(&mut *tx)
+ .await?;
+
+ if let Some(thread_id) = thread_id {
+ crate::threads::update::sync_thread_calendar_flag(tx, thread_id).await?;
+ }
+
Ok(())
}
diff --git a/rust/cloud-storage/email_db_client/src/attachments/provider/test.rs b/rust/cloud-storage/email_db_client/src/attachments/provider/test.rs
index 47a21b5ff6..77ecdf9a7e 100644
--- a/rust/cloud-storage/email_db_client/src/attachments/provider/test.rs
+++ b/rust/cloud-storage/email_db_client/src/attachments/provider/test.rs
@@ -199,3 +199,63 @@ async fn fetch_db_attachments_in_bulk_returns_empty_for_nonexistent_messages(
Ok(())
}
+
+async fn fetch_calendar_flag(pool: &Pool<Postgres>, thread_id: Uuid) -> Result<bool> {
+ Ok(sqlx::query_scalar!(
+ "SELECT has_calendar_attachment FROM email_threads WHERE id = $1",
+ thread_id
+ )
+ .fetch_one(pool)
+ .await?)
+}
+
+fn service_attachment(
+ filename: &str,
+ mime_type: &str,
+ provider_id: &str,
+) -> models_email::service::attachment::Attachment {
+ models_email::service::attachment::Attachment {
+ db_id: Uuid::new_v4(),
+ provider_id: Some(provider_id.to_string()),
+ data_url: None,
+ filename: Some(filename.to_string()),
+ mime_type: Some(mime_type.to_string()),
+ size_bytes: Some(1024),
+ sfs_id: None,
+ content_id: None,
+ }
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("sync_thread_calendar_flag"))
+)]
+async fn insert_attachments_sets_then_clears_calendar_flag(pool: Pool<Postgres>) -> Result<()> {
+ let thread_id = Uuid::parse_str("00000000-0000-0000-0000-00000000b203")?;
+ let message_id = Uuid::parse_str("00000000-0000-0000-0000-00000000b504")?;
+ assert!(!fetch_calendar_flag(&pool, thread_id).await?);
+
+ // Sync an .ics onto the message: the thread flag turns on.
+ let mut tx = pool.begin().await?;
+ let mut attachments = vec![service_attachment(
+ "invite.ics",
+ "text/calendar",
+ "prov-ics-1",
+ )];
+ crate::attachments::provider::insert_attachments(&mut tx, message_id, &mut attachments).await?;
+ tx.commit().await?;
+ assert!(fetch_calendar_flag(&pool, thread_id).await?);
+
+ // Re-sync with only a PDF: the .ics is orphan-deleted and the flag clears.
+ let mut tx = pool.begin().await?;
+ let mut attachments = vec![service_attachment(
+ "notes.pdf",
+ "application/pdf",
+ "prov-pdf-1",
+ )];
+ crate::attachments::provider::insert_attachments(&mut tx, message_id, &mut attachments).await?;
+ tx.commit().await?;
+ assert!(!fetch_calendar_flag(&pool, thread_id).await?);
+
+ Ok(())
+}
diff --git a/rust/cloud-storage/email_db_client/src/messages/delete.rs b/rust/cloud-storage/email_db_client/src/messages/delete.rs
index 733e553c2f..18618261e7 100644
--- a/rust/cloud-storage/email_db_client/src/messages/delete.rs
+++ b/rust/cloud-storage/email_db_client/src/messages/delete.rs
@@ -24,6 +24,13 @@ pub async fn delete_message_with_tx(
.await?;
}
+ // The message's attachments cascade with the delete, which may have
+ // removed the thread's last calendar attachment. Drafts never have
+ // email_attachments rows, so they can't move the flag.
+ if !deleted_thread && !message.is_draft {
+ threads::update::sync_thread_calendar_flag(&mut *tx, message.thread_db_id).await?;
+ }
+
if deleted_thread {
Ok(Some(message.thread_db_id))
} else {
diff --git a/rust/cloud-storage/email_db_client/src/messages/test.rs b/rust/cloud-storage/email_db_client/src/messages/test.rs
index 1d10b24238..d5e0877679 100644
--- a/rust/cloud-storage/email_db_client/src/messages/test.rs
+++ b/rust/cloud-storage/email_db_client/src/messages/test.rs
@@ -296,3 +296,49 @@ async fn get_scheduled_db_messages_returns_empty_for_nonexistent_link(
Ok(())
}
+
+async fn fetch_calendar_flag(pool: &Pool<Postgres>, thread_id: Uuid) -> anyhow::Result<bool> {
+ Ok(sqlx::query_scalar!(
+ "SELECT has_calendar_attachment FROM email_threads WHERE id = $1",
+ thread_id
+ )
+ .fetch_one(pool)
+ .await?)
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../fixtures", scripts("sync_thread_calendar_flag"))
+)]
+async fn delete_message_clears_calendar_flag_when_last_ics_message_removed(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let thread_id = Uuid::parse_str("00000000-0000-0000-0000-00000000b201")?;
+ let ics_message_id = Uuid::parse_str("00000000-0000-0000-0000-00000000b501")?;
+ let fusionauth_user_id = "00000000-0000-0000-0000-000000000b01";
+
+ // Establish the flag from the fixture's .ics attachment.
+ let mut conn = pool.acquire().await?;
+ crate::threads::update::sync_thread_calendar_flag(&mut conn, thread_id).await?;
+ drop(conn);
+ assert!(fetch_calendar_flag(&pool, thread_id).await?);
+
+ let message = crate::messages::get_simple_messages::get_simple_message(
+ &pool,
+ &ics_message_id,
+ fusionauth_user_id,
+ )
+ .await?
+ .expect("fixture message present");
+
+ let mut tx = pool.begin().await?;
+ let deleted_thread =
+ crate::messages::delete::delete_message_with_tx(&mut tx, &message, true).await?;
+ tx.commit().await?;
+
+ // The thread survives (a second message remains) and the flag flips off
+ // because its attachments cascaded away with the message.
+ assert!(deleted_thread.is_none());
+ assert!(!fetch_calendar_flag(&pool, thread_id).await?);
+ Ok(())
+}
diff --git a/rust/cloud-storage/email_db_client/src/threads/update.rs b/rust/cloud-storage/email_db_client/src/threads/update.rs
index 2a1348df46..09f23869b3 100644
--- a/rust/cloud-storage/email_db_client/src/threads/update.rs
+++ b/rust/cloud-storage/email_db_client/src/threads/update.rs
@@ -1,3 +1,6 @@
+#[cfg(test)]
+mod test;
+
use crate::messages::get::fetch_messages_metadata;
use chrono::{DateTime, Utc};
@@ -133,6 +136,40 @@ pub async fn update_thread_provider_id(
Ok(())
}
+/// Recomputes the denormalized `email_threads.has_calendar_attachment` flag
+/// from the thread's current attachment set. Mirrors the CalendarOnly
+/// predicate in the email crate's dynamic query builder.
+#[tracing::instrument(skip(tx), err)]
+pub async fn sync_thread_calendar_flag(
+ tx: &mut sqlx::PgConnection,
+ thread_db_id: Uuid,
+) -> anyhow::Result<()> {
+ sqlx::query!(
+ r#"
+ UPDATE email_threads t
+ SET has_calendar_attachment = calc.has_cal
+ FROM (
+ SELECT EXISTS (
+ SELECT 1
+ FROM email_messages m
+ JOIN email_attachments a ON a.message_id = m.id
+ WHERE m.thread_id = $1
+ AND (a.filename ILIKE '%.ics'
+ OR a.mime_type = 'text/calendar'
+ OR a.mime_type = 'application/ics')
+ ) AS has_cal
+ ) calc
+ WHERE t.id = $1
+ AND t.has_calendar_attachment IS DISTINCT FROM calc.has_cal
+ "#,
+ thread_db_id
+ )
+ .execute(tx)
+ .await?;
+
+ Ok(())
+}
+
// Updates a thread's metadata (archived status, latest_timestamps)
#[tracing::instrument(skip(tx), err)]
pub async fn update_thread_metadata(
diff --git a/rust/cloud-storage/email_db_client/src/threads/update/test.rs b/rust/cloud-storage/email_db_client/src/threads/update/test.rs
new file mode 100644
index 0000000000..7292f5a3da
--- /dev/null
+++ b/rust/cloud-storage/email_db_client/src/threads/update/test.rs
@@ -0,0 +1,64 @@
+use super::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::{Pool, Postgres};
+
+const THREAD_WITH_ICS: &str = "00000000-0000-0000-0000-00000000b201";
+const THREAD_STALE_TRUE_PDF_ONLY: &str = "00000000-0000-0000-0000-00000000b202";
+
+async fn fetch_flag(pool: &Pool<Postgres>, thread_id: &str) -> anyhow::Result<bool> {
+ Ok(sqlx::query_scalar!(
+ "SELECT has_calendar_attachment FROM email_threads WHERE id = $1",
+ Uuid::parse_str(thread_id)?
+ )
+ .fetch_one(pool)
+ .await?)
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("sync_thread_calendar_flag"))
+)]
+async fn sync_sets_flag_when_thread_has_calendar_attachment(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ assert!(!fetch_flag(&pool, THREAD_WITH_ICS).await?);
+
+ let mut conn = pool.acquire().await?;
+ sync_thread_calendar_flag(&mut conn, Uuid::parse_str(THREAD_WITH_ICS)?).await?;
+
+ assert!(fetch_flag(&pool, THREAD_WITH_ICS).await?);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("sync_thread_calendar_flag"))
+)]
+async fn sync_clears_stale_flag_when_no_calendar_attachment(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ // Fixture marks this thread true even though its only attachment is a PDF.
+ assert!(fetch_flag(&pool, THREAD_STALE_TRUE_PDF_ONLY).await?);
+
+ let mut conn = pool.acquire().await?;
+ sync_thread_calendar_flag(&mut conn, Uuid::parse_str(THREAD_STALE_TRUE_PDF_ONLY)?).await?;
+
+ assert!(!fetch_flag(&pool, THREAD_STALE_TRUE_PDF_ONLY).await?);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("sync_thread_calendar_flag"))
+)]
+async fn sync_is_stable_when_flag_already_matches(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let thread_id = Uuid::parse_str(THREAD_WITH_ICS)?;
+ let mut conn = pool.acquire().await?;
+ sync_thread_calendar_flag(&mut conn, thread_id).await?;
+ assert!(fetch_flag(&pool, THREAD_WITH_ICS).await?);
+
+ // Re-sync with no attachment changes: the value must stay put.
+ sync_thread_calendar_flag(&mut conn, thread_id).await?;
+ assert!(fetch_flag(&pool, THREAD_WITH_ICS).await?);
+ Ok(())
+}
diff --git a/rust/cloud-storage/email_service/Cargo.toml b/rust/cloud-storage/email_service/Cargo.toml
index 6481d89304..d3c70ef435 100644
--- a/rust/cloud-storage/email_service/Cargo.toml
+++ b/rust/cloud-storage/email_service/Cargo.toml
@@ -13,6 +13,10 @@ path = "src/main.rs"
name = "backfill_contacts"
path = "src/bin/backfill_contacts/backfill_contacts.rs"
+[[bin]]
+name = "backfill_calendar_flags"
+path = "src/bin/backfill_calendar_flags/backfill_calendar_flags.rs"
+
[[bin]]
name = "backfill_gmail_attachments"
path = "src/bin/backfill_gmail_attachments/backfill_gmail_attachments.rs"
diff --git a/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/backfill_calendar_flags.rs b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/backfill_calendar_flags.rs
new file mode 100644
index 0000000000..85b9decd2c
--- /dev/null
+++ b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/backfill_calendar_flags.rs
@@ -0,0 +1,102 @@
+//! # Calendar Flag Backfill Utility
+//!
+//! Backfills the denormalized `email_threads.has_calendar_attachment` column:
+//! for each user, flags every thread that has at least one iCalendar
+//! attachment (`*.ics` filename or `text/calendar` / `application/ics` mime).
+//! Only ever sets the flag to true — steady-state maintenance (including
+//! clearing it on deletes) is handled by the ingest and delete paths in
+//! `email_db_client`.
+//!
+//! ## Required Environment Variables:
+//! - `DATABASE_URL`: The connection string for the PostgreSQL database.
+//!
+//! ## Optional Environment Variables:
+//! - `MACRO_IDS`: Comma-separated macro IDs to backfill. When unset, every
+//! user with an email link is processed.
+//! - `CONCURRENCY`: Number of users processed concurrently (defaults to 10).
+
+mod config;
+mod process;
+
+use anyhow::Context;
+use futures::stream::{self, StreamExt};
+use macro_entrypoint::MacroEntrypoint;
+use sqlx::postgres::PgPoolOptions;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+ println!("Loading configuration...");
+ MacroEntrypoint::default().init();
+ let config = config::Config::from_env().context("Failed to load configuration")?;
+
+ println!("Connecting to the database...");
+ let db_pool = PgPoolOptions::new()
+ .min_connections(1)
+ .max_connections(config.concurrency as u32)
+ .connect(&config.database_url)
+ .await
+ .context("Could not connect to db")?;
+
+ let macro_ids: Vec<String> = match &config.macro_ids {
+ Some(ids) => ids.split(',').map(|id| id.trim().to_string()).collect(),
+ None => process::fetch_all_macro_ids(&db_pool)
+ .await
+ .context("Failed to fetch macro IDs")?,
+ };
+
+ let total_users = macro_ids.len();
+ println!(
+ "Processing {total_users} macro IDs with concurrency {}",
+ config.concurrency
+ );
+
+ let results: Vec<(String, anyhow::Result<u64>)> = stream::iter(macro_ids)
+ .map(|macro_id| {
+ let db_pool = db_pool.clone();
+ async move {
+ let result = process::process_macro_id(&db_pool, &macro_id).await;
+ (macro_id, result)
+ }
+ })
+ .buffer_unordered(config.concurrency)
+ .enumerate()
+ .map(|(index, (macro_id, result))| {
+ match &result {
+ Ok(flagged) => println!(
+ "=== Completed {macro_id} ({}/{total_users}): flagged {flagged} threads ===",
+ index + 1
+ ),
+ Err(e) => println!(
+ "=== Failed {macro_id} ({}/{total_users}): {e:?} ===",
+ index + 1
+ ),
+ }
+ (macro_id, result)
+ })
+ .collect()
+ .await;
+
+ let total_flagged: u64 = results.iter().filter_map(|(_, r)| r.as_ref().ok()).sum();
+ let failures: Vec<&String> = results
+ .iter()
+ .filter(|(_, r)| r.is_err())
+ .map(|(id, _)| id)
+ .collect();
+
+ println!(
+ "\n=== All macro IDs processed: {total_flagged} threads flagged, {} failures ===",
+ failures.len()
+ );
+ if !failures.is_empty() {
+ anyhow::bail!(
+ "{} macro IDs failed ({}); rerun to retry (already-flagged threads are skipped)",
+ failures.len(),
+ failures
+ .iter()
+ .map(|s| s.as_str())
+ .collect::<Vec<_>>()
+ .join(", ")
+ );
+ }
+ Ok(())
+}
diff --git a/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/config.rs b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/config.rs
new file mode 100644
index 0000000000..fa099457f7
--- /dev/null
+++ b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/config.rs
@@ -0,0 +1,41 @@
+use anyhow::Context;
+use macro_env_var::env_vars;
+
+/// Holds all configuration loaded from environment variables.
+pub struct Config {
+ /// Comma-separated macro IDs to backfill. When unset, every user with an
+ /// email link is processed.
+ pub macro_ids: Option<String>,
+ pub database_url: String,
+ /// Number of users processed concurrently. Defaults to 10, capped at 50
+ /// to bound DB connection usage.
+ pub concurrency: usize,
+}
+
+env_vars! {
+ struct MacroIds;
+ struct DatabaseUrl;
+ struct Concurrency;
+}
+
+impl Config {
+ /// Creates a new `Config` instance by reading from environment variables.
+ /// Returns an error if any required variable is not set.
+ pub fn from_env() -> anyhow::Result<Self> {
+ let concurrency = match Concurrency::new().ok() {
+ Some(v) => v
+ .parse::<usize>()
+ .context("CONCURRENCY is not a number")?
+ .clamp(1, 50),
+ None => 10,
+ };
+
+ Ok(Self {
+ macro_ids: MacroIds::new().ok().map(|v| v.to_string()),
+ database_url: DatabaseUrl::new()
+ .context("DATABASE_URL not set")?
+ .to_string(),
+ concurrency,
+ })
+ }
+}
diff --git a/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/process.rs b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/process.rs
new file mode 100644
index 0000000000..b36c20d472
--- /dev/null
+++ b/rust/cloud-storage/email_service/src/bin/backfill_calendar_flags/process.rs
@@ -0,0 +1,60 @@
+use sqlx::types::Uuid;
+
+/// Flags every thread on one of the user's links that has at least one
+/// calendar attachment. One statement per link keeps transactions small and
+/// the progress output per-mailbox. Returns the number of threads flagged.
+pub async fn process_macro_id(pool: &sqlx::PgPool, macro_id: &str) -> anyhow::Result<u64> {
+ let link_ids: Vec<Uuid> =
+ sqlx::query_scalar!("SELECT id FROM email_links WHERE macro_id = $1", macro_id)
+ .fetch_all(pool)
+ .await?;
+
+ if link_ids.is_empty() {
+ println!("No email links found for {macro_id}.");
+ return Ok(0);
+ }
+
+ let mut total_flagged = 0u64;
+ for link_id in link_ids {
+ // Mirrors the CalendarOnly predicate in the email crate's dynamic
+ // query builder and email_db_client::threads::update::sync_thread_calendar_flag.
+ let flagged = sqlx::query!(
+ r#"
+ UPDATE email_threads t
+ SET has_calendar_attachment = true
+ FROM (
+ SELECT DISTINCT m.thread_id
+ FROM email_messages m
+ JOIN email_attachments a ON a.message_id = m.id
+ WHERE m.link_id = $1
+ AND (a.filename ILIKE '%.ics'
+ OR a.mime_type = 'text/calendar'
+ OR a.mime_type = 'application/ics')
+ ) cal
+ WHERE t.id = cal.thread_id
+ AND NOT t.has_calendar_attachment
+ "#,
+ link_id
+ )
+ .execute(pool)
+ .await?
+ .rows_affected();
+
+ // Prefix with the user so interleaved concurrent output stays readable.
+ println!("[{macro_id}] link {link_id}: flagged {flagged} threads");
+ total_flagged += flagged;
+ }
+
+ Ok(total_flagged)
+}
+
+/// Every macro ID that owns at least one email link. Connected secondary
+/// mailboxes carry their own macro_id row in email_links, so iterating these
+/// covers every link exactly once.
+pub async fn fetch_all_macro_ids(pool: &sqlx::PgPool) -> anyhow::Result<Vec<String>> {
+ Ok(
+ sqlx::query_scalar!("SELECT DISTINCT macro_id FROM email_links ORDER BY macro_id")
+ .fetch_all(pool)
+ .await?,
+ )
+}
diff --git a/rust/cloud-storage/macro_db_client/migrations/20260706155906_add_email_threads_has_calendar_attachment.sql b/rust/cloud-storage/macro_db_client/migrations/20260706155906_add_email_threads_has_calendar_attachment.sql
new file mode 100644
index 0000000000..e157907d7d
--- /dev/null
+++ b/rust/cloud-storage/macro_db_client/migrations/20260706155906_add_email_threads_has_calendar_attachment.sql
@@ -0,0 +1,13 @@
+-- Denormalized "thread has a calendar attachment" flag for the CalendarOnly
+-- soup filter. Deriving it at query time joins email_attachments against
+-- email_messages (hundreds of thousands of rows per link) on every page load.
+-- Maintained at attachment ingest and message delete (email_db_client);
+-- existing threads are flagged by the backfill_calendar_flags util in
+-- email_service.
+ALTER TABLE email_threads
+ ADD COLUMN IF NOT EXISTS has_calendar_attachment boolean NOT NULL DEFAULT false;
+
+-- Superseded by idx_email_threads_calendar_view (next migration), which adds
+-- the sort key. CREATE INDEX CONCURRENTLY can't run in a transaction, so it
+-- lives in its own no-transaction migration.
+DROP INDEX IF EXISTS idx_email_threads_calendar_link_id;
diff --git a/rust/cloud-storage/macro_db_client/migrations/20260706175458_email_threads_calendar_view_index.sql b/rust/cloud-storage/macro_db_client/migrations/20260706175458_email_threads_calendar_view_index.sql
new file mode 100644
index 0000000000..2e7eab2df9
--- /dev/null
+++ b/rust/cloud-storage/macro_db_client/migrations/20260706175458_email_threads_calendar_view_index.sql
@@ -0,0 +1,8 @@
+-- no-transaction
+-- Matches the CalendarOnly candidate scan's ORDER BY
+-- (COALESCE(latest_non_spam_message_ts, updated_at) DESC, id DESC) so
+-- per-link ordered scans stop at the LIMIT instead of fetching and sorting
+-- every flagged thread.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_email_threads_calendar_view
+ ON email_threads (link_id, (COALESCE(latest_non_spam_message_ts, updated_at)) DESC, id DESC)
+ WHERE has_calendar_attachment;