Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(email): read importance from the denormalized is_signal flag (#4673)

Evan Hutnikcf2fe1b8773a
diff --git a/rust/cloud-storage/.sqlx/query-039dd65419a635365c37d46699bf115e771ceae4526e0a13928152e2bb1bb9b3.json b/rust/cloud-storage/.sqlx/query-039dd65419a635365c37d46699bf115e771ceae4526e0a13928152e2bb1bb9b3.json
deleted file mode 100644
index 887d1272a9..0000000000
--- a/rust/cloud-storage/.sqlx/query-039dd65419a635365c37d46699bf115e771ceae4526e0a13928152e2bb1bb9b3.json
+++ /dev/null
@@ -1,107 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT\n t.id as thread_id,\n -- Return timestamps from the chosen messages directly\n COALESCE(earliest_msg.sent_at, earliest_msg.updated_at) as \"first_message_ts!\",\n COALESCE(latest_msg.sent_at, latest_msg.updated_at) as \"last_message_ts!\",\n -- Last time user viewed this thread\n uh.updated_at as \"viewed_at?\",\n latest_msg.snippet,\n earliest_msg.subject as \"subject?\",\n l.macro_id,\n t.link_id,\n latest_msg.sender as sender,\n latest_msg.pretty_sender as \"pretty_sender!\",\n latest_msg.trash_label as trash_label,\n (\n SELECT m_latest.is_draft\n FROM email_messages m_latest\n WHERE m_latest.thread_id = t.id\n AND m_latest.link_id = t.link_id\n ORDER BY m_latest.internal_date_ts DESC NULLS LAST\n LIMIT 1\n ) AS \"is_draft!\",\n t.is_read,\n t.inbox_visible,\n (\n SELECT EXISTS (\n SELECT 1\n FROM email_messages m_imp\n JOIN email_message_labels ml ON m_imp.id = ml.message_id\n JOIN email_labels l ON ml.label_id = l.id\n WHERE m_imp.thread_id = t.id\n AND l.link_id = t.link_id\n AND l.name = 'IMPORTANT'\n )\n ) AS \"is_important!\"\n FROM email_threads t\n LEFT JOIN email_user_history uh ON uh.thread_id = t.id AND uh.link_id = t.link_id\n LEFT JOIN email_links l ON l.id = t.link_id\n -- LATERAL join for LATEST message\n -- JOIN (Inner) acts as a filter to ensure we only return threads that actually have messages\n JOIN LATERAL (\n SELECT\n m2.snippet,\n m2.sent_at,\n m2.updated_at,\n c.email_address as sender,\n COALESCE(c.name, c.email_address) as pretty_sender,\n EXISTS(\n SELECT 1 \n FROM email_message_labels eml \n JOIN email_labels el ON el.id = eml.label_id \n WHERE eml.message_id = m2.id \n AND el.provider_label_id = 'TRASH'\n ) as trash_label\n FROM email_messages m2\n LEFT JOIN email_contacts c ON c.id = m2.from_contact_id\n WHERE m2.thread_id = t.id\n AND m2.link_id = t.link_id\n ORDER BY\n (CASE WHEN m2.is_draft = false AND m2.sent_at IS NOT NULL THEN 0 ELSE 1 END) ASC,\n COALESCE(m2.sent_at, m2.updated_at) DESC NULLS LAST\n LIMIT 1\n ) latest_msg ON true\n -- LATERAL join for EARLIEST message\n LEFT JOIN LATERAL (\n SELECT\n m3.subject,\n m3.sent_at,\n m3.updated_at\n FROM email_messages m3\n WHERE m3.thread_id = t.id\n AND m3.link_id = t.link_id\n ORDER BY\n -- 1. Priority: Non-drafts with valid sent_at come first (0), everything else is fallback (1)\n (CASE WHEN m3.is_draft = false AND m3.sent_at IS NOT NULL THEN 0 ELSE 1 END) ASC,\n -- 2. Sort by time (Oldest first)\n COALESCE(m3.sent_at, m3.updated_at) ASC NULLS LAST\n LIMIT 1\n ) earliest_msg ON true\n WHERE t.id = ANY($2)\n AND t.link_id = ANY($1)\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "thread_id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 1,
- "name": "first_message_ts!",
- "type_info": "Timestamptz"
- },
- {
- "ordinal": 2,
- "name": "last_message_ts!",
- "type_info": "Timestamptz"
- },
- {
- "ordinal": 3,
- "name": "viewed_at?",
- "type_info": "Timestamptz"
- },
- {
- "ordinal": 4,
- "name": "snippet",
- "type_info": "Text"
- },
- {
- "ordinal": 5,
- "name": "subject?",
- "type_info": "Text"
- },
- {
- "ordinal": 6,
- "name": "macro_id",
- "type_info": "Text"
- },
- {
- "ordinal": 7,
- "name": "link_id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 8,
- "name": "sender",
- "type_info": "Varchar"
- },
- {
- "ordinal": 9,
- "name": "pretty_sender!",
- "type_info": "Varchar"
- },
- {
- "ordinal": 10,
- "name": "trash_label",
- "type_info": "Bool"
- },
- {
- "ordinal": 11,
- "name": "is_draft!",
- "type_info": "Bool"
- },
- {
- "ordinal": 12,
- "name": "is_read",
- "type_info": "Bool"
- },
- {
- "ordinal": 13,
- "name": "inbox_visible",
- "type_info": "Bool"
- },
- {
- "ordinal": 14,
- "name": "is_important!",
- "type_info": "Bool"
- }
- ],
- "parameters": {
- "Left": [
- "UuidArray",
- "UuidArray"
- ]
- },
- "nullable": [
- false,
- null,
- null,
- false,
- true,
- true,
- false,
- false,
- false,
- null,
- null,
- null,
- false,
- false,
- null
- ]
- },
- "hash": "039dd65419a635365c37d46699bf115e771ceae4526e0a13928152e2bb1bb9b3"
-}
diff --git a/rust/cloud-storage/.sqlx/query-3b9b441b224f60a4832dee29e4151a2bc6a87fe87544bf4fdfeee20aeec7c5bf.json b/rust/cloud-storage/.sqlx/query-3b9b441b224f60a4832dee29e4151a2bc6a87fe87544bf4fdfeee20aeec7c5bf.json
new file mode 100644
index 0000000000..3a34936eb7
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-3b9b441b224f60a4832dee29e4151a2bc6a87fe87544bf4fdfeee20aeec7c5bf.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT count(*) AS \"mismatched!\"\n FROM email_threads t\n WHERE t.link_id = $1\n AND t.is_signal IS DISTINCT FROM EXISTS (\n SELECT 1\n FROM email_messages m\n WHERE m.thread_id = t.id\n AND NOT EXISTS (\n SELECT 1 FROM email_message_labels ml\n JOIN email_labels l ON ml.label_id = l.id\n WHERE ml.message_id = m.id AND l.name = 'TRASH'\n )\n AND (\n (\n EXISTS (\n SELECT 1\n FROM email_contacts sender_c\n JOIN email_filters ef\n ON ef.link_id = m.link_id\n AND ef.email_address IS NOT NULL\n AND LOWER(ef.email_address) = LOWER(sender_c.email_address)\n WHERE sender_c.id = m.from_contact_id\n AND ef.is_important = TRUE\n )\n OR EXISTS (\n SELECT 1\n FROM email_contacts sender_c\n JOIN email_filters ef\n ON ef.link_id = m.link_id\n AND ef.email_domain IS NOT NULL\n AND LOWER(ef.email_domain) = LOWER(SPLIT_PART(sender_c.email_address, '@', 2))\n WHERE sender_c.id = m.from_contact_id\n AND ef.is_important = TRUE\n AND NOT EXISTS (\n SELECT 1 FROM email_filters ef_addr\n WHERE ef_addr.link_id = m.link_id\n AND ef_addr.email_address IS NOT NULL\n AND LOWER(ef_addr.email_address) = LOWER(sender_c.email_address)\n AND ef_addr.is_important = FALSE\n )\n )\n )\n OR (\n NOT (\n EXISTS (\n SELECT 1\n FROM email_contacts sender_c\n JOIN email_filters ef\n ON ef.link_id = m.link_id\n AND ef.email_address IS NOT NULL\n AND LOWER(ef.email_address) = LOWER(sender_c.email_address)\n WHERE sender_c.id = m.from_contact_id\n AND ef.is_important = FALSE\n )\n OR EXISTS (\n SELECT 1\n FROM email_contacts sender_c\n JOIN email_filters ef\n ON ef.link_id = m.link_id\n AND ef.email_domain IS NOT NULL\n AND LOWER(ef.email_domain) = LOWER(SPLIT_PART(sender_c.email_address, '@', 2))\n WHERE sender_c.id = m.from_contact_id\n AND ef.is_important = FALSE\n AND NOT EXISTS (\n SELECT 1 FROM email_filters ef_addr\n WHERE ef_addr.link_id = m.link_id\n AND ef_addr.email_address IS NOT NULL\n AND LOWER(ef_addr.email_address) = LOWER(sender_c.email_address)\n AND ef_addr.is_important = TRUE\n )\n )\n )\n AND (\n m.is_draft = TRUE\n OR EXISTS (\n SELECT 1 FROM email_message_labels ml\n JOIN email_labels l ON ml.label_id = l.id\n WHERE ml.message_id = m.id\n AND l.name IN ('CATEGORY_PERSONAL', 'SENT', 'DRAFT')\n )\n OR NOT EXISTS (\n SELECT 1 FROM email_message_labels ml\n JOIN email_labels l ON ml.label_id = l.id\n WHERE ml.message_id = m.id\n AND l.name IN ('CATEGORY_UPDATES', 'CATEGORY_PROMOTIONS', 'CATEGORY_SOCIAL', 'CATEGORY_FORUMS')\n )\n )\n )\n )\n )\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "mismatched!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "3b9b441b224f60a4832dee29e4151a2bc6a87fe87544bf4fdfeee20aeec7c5bf"
+}
diff --git a/rust/cloud-storage/.sqlx/query-e3effaf90fa2b1bf1dc615ce0c74c1afc8d0e6774e71862ee0d39b5381647308.json b/rust/cloud-storage/.sqlx/query-e3effaf90fa2b1bf1dc615ce0c74c1afc8d0e6774e71862ee0d39b5381647308.json
new file mode 100644
index 0000000000..8919fd97da
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-e3effaf90fa2b1bf1dc615ce0c74c1afc8d0e6774e71862ee0d39b5381647308.json
@@ -0,0 +1,107 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n t.id as thread_id,\n -- Return timestamps from the chosen messages directly\n COALESCE(earliest_msg.sent_at, earliest_msg.updated_at) as \"first_message_ts!\",\n COALESCE(latest_msg.sent_at, latest_msg.updated_at) as \"last_message_ts!\",\n -- Last time user viewed this thread\n uh.updated_at as \"viewed_at?\",\n latest_msg.snippet,\n earliest_msg.subject as \"subject?\",\n l.macro_id,\n t.link_id,\n latest_msg.sender as sender,\n latest_msg.pretty_sender as \"pretty_sender!\",\n latest_msg.trash_label as trash_label,\n (\n SELECT m_latest.is_draft\n FROM email_messages m_latest\n WHERE m_latest.thread_id = t.id\n AND m_latest.link_id = t.link_id\n ORDER BY m_latest.internal_date_ts DESC NULLS LAST\n LIMIT 1\n ) AS \"is_draft!\",\n t.is_read,\n t.inbox_visible,\n -- Gmail's IMPORTANT label — a different notion from the\n -- Importance filter / email_threads.is_signal heuristic.\n (\n SELECT EXISTS (\n SELECT 1\n FROM email_messages m_imp\n JOIN email_message_labels ml ON m_imp.id = ml.message_id\n JOIN email_labels l ON ml.label_id = l.id\n WHERE m_imp.thread_id = t.id\n AND l.link_id = t.link_id\n AND l.name = 'IMPORTANT'\n )\n ) AS \"is_important!\"\n FROM email_threads t\n LEFT JOIN email_user_history uh ON uh.thread_id = t.id AND uh.link_id = t.link_id\n LEFT JOIN email_links l ON l.id = t.link_id\n -- LATERAL join for LATEST message\n -- JOIN (Inner) acts as a filter to ensure we only return threads that actually have messages\n JOIN LATERAL (\n SELECT\n m2.snippet,\n m2.sent_at,\n m2.updated_at,\n c.email_address as sender,\n COALESCE(c.name, c.email_address) as pretty_sender,\n EXISTS(\n SELECT 1 \n FROM email_message_labels eml \n JOIN email_labels el ON el.id = eml.label_id \n WHERE eml.message_id = m2.id \n AND el.provider_label_id = 'TRASH'\n ) as trash_label\n FROM email_messages m2\n LEFT JOIN email_contacts c ON c.id = m2.from_contact_id\n WHERE m2.thread_id = t.id\n AND m2.link_id = t.link_id\n ORDER BY\n (CASE WHEN m2.is_draft = false AND m2.sent_at IS NOT NULL THEN 0 ELSE 1 END) ASC,\n COALESCE(m2.sent_at, m2.updated_at) DESC NULLS LAST\n LIMIT 1\n ) latest_msg ON true\n -- LATERAL join for EARLIEST message\n LEFT JOIN LATERAL (\n SELECT\n m3.subject,\n m3.sent_at,\n m3.updated_at\n FROM email_messages m3\n WHERE m3.thread_id = t.id\n AND m3.link_id = t.link_id\n ORDER BY\n -- 1. Priority: Non-drafts with valid sent_at come first (0), everything else is fallback (1)\n (CASE WHEN m3.is_draft = false AND m3.sent_at IS NOT NULL THEN 0 ELSE 1 END) ASC,\n -- 2. Sort by time (Oldest first)\n COALESCE(m3.sent_at, m3.updated_at) ASC NULLS LAST\n LIMIT 1\n ) earliest_msg ON true\n WHERE t.id = ANY($2)\n AND t.link_id = ANY($1)\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "thread_id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "first_message_ts!",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 2,
+ "name": "last_message_ts!",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 3,
+ "name": "viewed_at?",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 4,
+ "name": "snippet",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 5,
+ "name": "subject?",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 6,
+ "name": "macro_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 7,
+ "name": "link_id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 8,
+ "name": "sender",
+ "type_info": "Varchar"
+ },
+ {
+ "ordinal": 9,
+ "name": "pretty_sender!",
+ "type_info": "Varchar"
+ },
+ {
+ "ordinal": 10,
+ "name": "trash_label",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 11,
+ "name": "is_draft!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 12,
+ "name": "is_read",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 13,
+ "name": "inbox_visible",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 14,
+ "name": "is_important!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "UuidArray"
+ ]
+ },
+ "nullable": [
+ false,
+ null,
+ null,
+ false,
+ true,
+ true,
+ false,
+ false,
+ false,
+ null,
+ null,
+ null,
+ false,
+ false,
+ null
+ ]
+ },
+ "hash": "e3effaf90fa2b1bf1dc615ce0c74c1afc8d0e6774e71862ee0d39b5381647308"
+}
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs
index 437cdd51d8..9c00b1a369 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs
@@ -35,6 +35,7 @@ pub(super) fn has_thread_literals(ast: &Expr<EmailLiteral>) -> bool {
| EmailLiteral::Owner(_)
| EmailLiteral::ProjectId(_)
| EmailLiteral::CalendarOnly(_)
+ | EmailLiteral::Importance(_)
| EmailLiteral::CreatedAt(_)
| EmailLiteral::UpdatedAt(_)
| EmailLiteral::Property(_),
@@ -54,6 +55,7 @@ pub(super) fn has_message_literals(ast: &Expr<EmailLiteral>) -> bool {
| EmailLiteral::ProjectId(_)
| EmailLiteral::Shared(_)
| EmailLiteral::CalendarOnly(_)
+ | EmailLiteral::Importance(_)
| EmailLiteral::CreatedAt(_)
| EmailLiteral::UpdatedAt(_)
| EmailLiteral::Property(_),
@@ -62,15 +64,6 @@ pub(super) fn has_message_literals(ast: &Expr<EmailLiteral>) -> bool {
})
}
-pub(super) fn has_importance_literal(ast: &Expr<EmailLiteral>) -> bool {
- ast.collapse_frames(|frame| match frame {
- filter_ast::ExprFrame::And(a, b) | filter_ast::ExprFrame::Or(a, b) => a || b,
- filter_ast::ExprFrame::Not(a) => a,
- filter_ast::ExprFrame::Literal(EmailLiteral::Importance(_)) => true,
- filter_ast::ExprFrame::Literal(_) => false,
- })
-}
-
#[derive(Clone, Copy)]
enum AddressKind {
Sender,
@@ -184,44 +177,6 @@ fn build_address_text_match(
}
}
-fn build_sender_importance_override_filter(is_important: bool) -> SqlFragment {
- let importance_literal = if is_important { "TRUE" } else { "FALSE" };
- let opposite_importance_literal = if is_important { "FALSE" } else { "TRUE" };
-
- SqlFragment::raw(format!(
- r#"(
- EXISTS (
- SELECT 1
- FROM email_contacts sender_c
- JOIN email_filters ef
- ON ef.link_id = m.link_id
- AND ef.email_address IS NOT NULL
- AND LOWER(ef.email_address) = LOWER(sender_c.email_address)
- WHERE sender_c.id = m.from_contact_id
- AND ef.is_important = {importance_literal}
- )
- OR EXISTS (
- SELECT 1
- FROM email_contacts sender_c
- JOIN email_filters ef
- ON ef.link_id = m.link_id
- AND ef.email_domain IS NOT NULL
- AND LOWER(ef.email_domain) = LOWER(split_part(sender_c.email_address, '@', 2))
- WHERE sender_c.id = m.from_contact_id
- AND ef.is_important = {importance_literal}
- AND NOT EXISTS (
- SELECT 1
- FROM email_filters ef_addr
- WHERE ef_addr.link_id = m.link_id
- AND ef_addr.email_address IS NOT NULL
- AND LOWER(ef_addr.email_address) = LOWER(sender_c.email_address)
- AND ef_addr.is_important = {opposite_importance_literal}
- )
- )
- )"#,
- ))
-}
-
pub(super) fn build_message_email_filter(
ast: &Expr<EmailLiteral>,
resolved: &ResolvedFilters,
@@ -232,9 +187,7 @@ pub(super) fn build_message_email_filter(
filter_ast::ExprFrame::Not(a) => SqlFragment::not(a),
filter_ast::ExprFrame::Literal(
- EmailLiteral::ThreadId(_)
- | EmailLiteral::Owner(_)
- | EmailLiteral::ProjectId(_),
+ EmailLiteral::ThreadId(_) | EmailLiteral::Owner(_) | EmailLiteral::ProjectId(_),
) => SqlFragment::raw("TRUE"),
filter_ast::ExprFrame::Literal(EmailLiteral::Sender(email)) => {
@@ -253,79 +206,7 @@ pub(super) fn build_message_email_filter(
build_address_predicate_on_m(AddressKind::Bcc, &email, resolved)
}
- filter_ast::ExprFrame::Literal(EmailLiteral::Importance(true)) => {
- let mut f = SqlFragment::raw(
- r#"(
- NOT EXISTS (
- SELECT 1 FROM email_message_labels ml
- JOIN email_labels l ON ml.label_id = l.id
- WHERE ml.message_id = m.id
- AND l.name = 'TRASH'
- )
- AND (
- "#,
- );
- f.extend(build_sender_importance_override_filter(true));
- f.push_raw(
- r#"
- OR (
- NOT "#,
- );
- f.extend(build_sender_importance_override_filter(false));
- f.push_raw(
- r#"
- AND (
- m.is_draft = TRUE
- OR EXISTS (
- SELECT 1 FROM email_message_labels ml
- JOIN email_labels l ON ml.label_id = l.id
- WHERE ml.message_id = m.id
- AND l.name IN ('CATEGORY_PERSONAL', 'SENT', 'DRAFT')
- )
- OR NOT EXISTS (
- SELECT 1 FROM email_message_labels ml
- JOIN email_labels l ON ml.label_id = l.id
- WHERE ml.message_id = m.id
- AND l.name IN ('CATEGORY_UPDATES', 'CATEGORY_PROMOTIONS', 'CATEGORY_SOCIAL', 'CATEGORY_FORUMS')
- )
- )
- )
- )
- )"#,
- );
- f
- }
- filter_ast::ExprFrame::Literal(EmailLiteral::Importance(false)) => {
- let mut f = SqlFragment::raw(
- r#"(
- "#,
- );
- f.extend(build_sender_importance_override_filter(false));
- f.push_raw(
- r#"
- OR (
- NOT "#,
- );
- f.extend(build_sender_importance_override_filter(true));
- f.push_raw(
- r#"
- AND NOT EXISTS (
- SELECT 1 FROM email_message_labels ml
- JOIN email_labels l ON ml.label_id = l.id
- WHERE ml.message_id = m.id
- AND l.name IN ('CATEGORY_PERSONAL', 'SENT', 'DRAFT')
- )
- AND EXISTS (
- SELECT 1 FROM email_message_labels ml
- JOIN email_labels l ON ml.label_id = l.id
- WHERE ml.message_id = m.id
- AND l.name IN ('CATEGORY_UPDATES', 'CATEGORY_PROMOTIONS', 'CATEGORY_SOCIAL', 'CATEGORY_FORUMS')
- )
- )
- )"#,
- );
- f
- }
+ filter_ast::ExprFrame::Literal(EmailLiteral::Importance(_)) => SqlFragment::raw("TRUE"),
filter_ast::ExprFrame::Literal(EmailLiteral::NotificationDone(_)) => {
SqlFragment::raw("TRUE")
}
@@ -360,7 +241,7 @@ pub(super) fn has_address_literals(ast: &Expr<EmailLiteral>) -> bool {
/// True if the subtree contains only address literals (Sender/Cc/Bcc/Recipient)
/// composed via And/Or/Not. Used to decide whether a top-level conjunct can be
/// safely pushed into the candidate-thread pre-filter without risking false
-/// negatives (e.g., `Sender(X) OR Importance(true)` cannot be reduced to just
+/// negatives (e.g., `Sender(X) OR CalendarOnly(true)` cannot be reduced to just
/// `Sender(X)` at the candidate stage).
fn is_pure_address_subtree(expr: &Expr<EmailLiteral>) -> bool {
expr.collapse_frames(|frame| match frame {
@@ -473,13 +354,12 @@ pub(super) fn build_thread_address_filter(ast: &Expr<EmailLiteral>) -> SqlFragme
/// True when the candidate WHERE must mirror the CROSS JOIN LATERAL's
/// message match via a correlated EXISTS rather than the address-only
/// `matching_threads` CTE. That's the case when the lateral applies a
-/// per-message filter the address CTE doesn't model — an `Importance`
-/// literal, or a non-empty view-level message filter (Starred / Drafts /
-/// Important / …). Without this, the candidate `LIMIT` counts threads that
-/// the lateral later drops, so the page under-fills while still emitting a
-/// cursor.
-pub(super) fn wants_message_exists_pushdown(ast: &Expr<EmailLiteral>, view: &PreviewView) -> bool {
- has_importance_literal(ast) || !build_view_message_filter(view).is_empty()
+/// per-message filter the address CTE doesn't model — a non-empty
+/// view-level message filter (Starred / Drafts / Important / …). Without
+/// this, the candidate `LIMIT` counts threads that the lateral later
+/// drops, so the page under-fills while still emitting a cursor.
+pub(super) fn wants_message_exists_pushdown(view: &PreviewView) -> bool {
+ !build_view_message_filter(view).is_empty()
}
/// Builds the candidate-stage mirror of the lateral message match:
@@ -775,6 +655,16 @@ pub(super) fn build_thread_email_filter(
SqlFragment::raw("TRUE")
}
+ // Denormalized importance flag maintained by update_thread_metadata
+ // (sync_thread_signal_flag) and the email_filters resync fan-out.
+ filter_ast::ExprFrame::Literal(EmailLiteral::Importance(true)) => {
+ SqlFragment::raw("t.is_signal")
+ }
+
+ filter_ast::ExprFrame::Literal(EmailLiteral::Importance(false)) => {
+ SqlFragment::raw("(NOT t.is_signal)")
+ }
+
filter_ast::ExprFrame::Literal(EmailLiteral::CreatedAt(ref lit)) => {
date_predicate("t.created_at", lit)
}
@@ -792,7 +682,6 @@ pub(super) fn build_thread_email_filter(
| EmailLiteral::Cc(_)
| EmailLiteral::Bcc(_)
| EmailLiteral::Recipient(_)
- | EmailLiteral::Importance(_)
| EmailLiteral::NotificationDone(_)
| EmailLiteral::NotificationSeen(_)
| EmailLiteral::Shared(_),
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs
index 5b9f7ae26c..23a07da1a5 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs
@@ -304,12 +304,12 @@ fn push_thread_candidate_select(
}
// Ensure the candidate LIMIT only counts threads that will survive the
- // CROSS JOIN LATERAL's message match. When a per-message filter is in
- // play (importance, or a view-level message filter) mirror the full
- // lateral predicate as a correlated EXISTS; otherwise push address-only
- // constraints through the index-driven `matching_threads` CTE referenced
- // via `t.id IN (SELECT thread_id FROM matching_threads)`.
- if wants_message_exists_pushdown(email_filter, view) {
+ // CROSS JOIN LATERAL's message match. When a view-level message filter
+ // is in play mirror the full lateral predicate as a correlated EXISTS;
+ // otherwise push address-only constraints through the index-driven
+ // `matching_threads` CTE referenced via
+ // `t.id IN (SELECT thread_id FROM matching_threads)`.
+ if wants_message_exists_pushdown(view) {
build_thread_message_exists_filter(email_filter, view, &params.resolved).push_into(builder);
} else if has_address_literals(email_filter) {
build_thread_address_filter(email_filter).push_into(builder);
@@ -406,11 +406,10 @@ fn build_query(
let defer_uh = !sort_uses_view_history(&params.sort_method_str);
let needs_shared_cte = !matches!(params.shared, SharedEmailFilter::Exclude);
- // When the candidate stage pushes a full per-message EXISTS (importance,
- // or a view-level message filter), it already enforces "thread has a
- // message the lateral will surface", so the address-only
- // `matching_threads` CTE is redundant.
- let matching_threads_body = if wants_message_exists_pushdown(email_filter, view) {
+ // When the candidate stage pushes a full per-message EXISTS (view-level
+ // message filter), it already enforces "thread has a message the lateral
+ // will surface", so the address-only `matching_threads` CTE is redundant.
+ let matching_threads_body = if wants_message_exists_pushdown(view) {
None
} else {
// Owned-only, non-team queries can scope the CTE's contact/message
@@ -464,6 +463,9 @@ fn build_query(
builder.push("t.viewed_at,");
}
+ // The is_important output column reflects Gmail's IMPORTANT label — a
+ // different notion from the Importance filter, which reads the
+ // denormalized email_threads.is_signal heuristic.
builder.push(
r#"
t.project_id,
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs
index 24466d7bd3..f77c394f95 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs
@@ -254,49 +254,56 @@ fn test_partial_and_domain_emit_different_predicates_for_same_string() {
}
#[test]
-fn test_build_message_email_filter_importance_true_includes_drafts() {
- let expr = Expr::Literal(EmailLiteral::Importance(true));
- let result = build_message_email_filter(&expr, &ResolvedFilters::empty());
- let debug = result.to_debug_sql();
-
- assert!(debug.contains("m.is_draft = TRUE"));
+fn test_importance_compiles_to_thread_signal_flag() {
+ let thread = build_thread_email_filter(
+ &Expr::Literal(EmailLiteral::Importance(true)),
+ "t.updated_at",
+ )
+ .to_debug_sql();
+ assert!(thread.contains("t.is_signal"));
+ assert!(!thread.contains("NOT t.is_signal"));
+
+ let thread = build_thread_email_filter(
+ &Expr::Literal(EmailLiteral::Importance(false)),
+ "t.updated_at",
+ )
+ .to_debug_sql();
+ assert!(thread.contains("NOT t.is_signal"));
+}
+
+#[test]
+fn test_importance_is_noop_in_message_filter() {
+ // The heuristic lives in the denormalized flag now; the lateral must not
+ // re-evaluate it per message.
+ for imp in [true, false] {
+ let result = build_message_email_filter(
+ &Expr::Literal(EmailLiteral::Importance(imp)),
+ &ResolvedFilters::empty(),
+ );
+ let debug = result.to_debug_sql();
+ assert!(!debug.contains("email_filters"));
+ assert!(!debug.contains("CATEGORY"));
+ assert!(!debug.contains("is_important"));
+ }
}
#[test]
-fn test_build_message_email_filter_importance_true_excludes_trash() {
- let expr = Expr::Literal(EmailLiteral::Importance(true));
- let result = build_message_email_filter(&expr, &ResolvedFilters::empty());
- let debug = result.to_debug_sql();
-
- assert!(debug.contains("l.name = 'TRASH'"));
- assert!(debug.contains("NOT EXISTS"));
-}
+fn test_importance_query_has_no_message_exists_pushdown() {
+ use item_filters::SharedEmailFilter;
-#[test]
-fn test_build_message_email_filter_importance_true_includes_email_filters() {
- let expr = Expr::Literal(EmailLiteral::Importance(true));
- let result = build_message_email_filter(&expr, &ResolvedFilters::empty());
- let debug = result.to_debug_sql();
-
- assert!(debug.contains("FROM email_filters ef"));
- assert!(
- debug
- .contains("LOWER(ef.email_domain) = LOWER(split_part(sender_c.email_address, '@', 2))")
+ // The Signal-tab shape: inbox view + Importance(true) + Shared(exclude).
+ // The candidate stage must filter on t.is_signal directly, with no
+ // per-message EXISTS mirror (the inbox view has no message filter).
+ let view = PreviewView::StandardLabel(PreviewViewStandardLabel::Inbox);
+ let expr = Expr::and(
+ Expr::Literal(EmailLiteral::Importance(true)),
+ Expr::Literal(EmailLiteral::Shared(SharedEmailFilter::Exclude)),
);
- assert!(debug.contains("ef.is_important = TRUE"));
- assert!(debug.contains("ef_addr.is_important = FALSE"));
-}
-
-#[test]
-fn test_build_message_email_filter_importance_false_includes_email_filters() {
- let expr = Expr::Literal(EmailLiteral::Importance(false));
- let result = build_message_email_filter(&expr, &ResolvedFilters::empty());
- let debug = result.to_debug_sql();
+ let sql = super::query::debug_build_query_sql(&view, &expr);
- assert!(debug.contains("FROM email_filters ef"));
- assert!(debug.contains("LOWER(ef.email_address) = LOWER(sender_c.email_address)"));
- assert!(debug.contains("ef.is_important = FALSE"));
- assert!(debug.contains("ef_addr.is_important = TRUE"));
+ assert!(sql.contains("t.is_signal"));
+ assert!(!sql.contains("AND EXISTS (SELECT 1 FROM email_messages m WHERE m.thread_id = t.id"));
+ assert!(!sql.contains("is_important = "));
}
#[test]
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/dynamic_query.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/dynamic_query.rs
index 31e1c6c72e..9d1e0b135c 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/dynamic_query.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/dynamic_query.rs
@@ -680,6 +680,7 @@ async fn test_dynamic_query_pagination(pool: Pool<Postgres>) -> anyhow::Result<(
fixtures(path = "../../../../fixtures", scripts("email_dynamic_query"))
)]
async fn test_dynamic_query_with_importance_filter(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);
let limit = 50;
@@ -748,8 +749,10 @@ async fn test_dynamic_query_with_importance_filter(pool: Pool<Postgres>) -> anyh
dynamic::dynamic_email_thread_cursor(&pool, &[link_id], limit, &view, query, "", None)
.await?;
- // Threads 6 (CATEGORY_UPDATES) and 7 (CATEGORY_PROMOTIONS)
- // Thread 8 has CATEGORY_UPDATES but is excluded because DRAFT is a priority label
+ // Threads 6 (CATEGORY_UPDATES) and 7 (CATEGORY_PROMOTIONS). The
+ // all-trash threads 9-11 are is_signal=false too, but the lateral's
+ // TRASH exclusion leaves them no message to surface, so they still
+ // drop out. Thread 8 is excluded because DRAFT made it signal.
assert_eq!(
results.len(),
2,
@@ -776,6 +779,53 @@ async fn test_dynamic_query_with_importance_filter(pool: Pool<Postgres>) -> anyh
Ok(())
}
+// KNOWN LIMITATION (pinned, not desired): an OR mixing a message-level
+// literal (Sender) with a thread-level literal (Importance) broadens to
+// match-everything. The thread stage maps Sender to TRUE and the lateral
+// maps Importance to TRUE, so `(TRUE OR t.is_signal)` / `(sender OR TRUE)`
+// both collapse. The correct result here would be the 6 signal threads
+// (john's threads 1/2/5 are all signal), but noise threads 6 and 7 leak in.
+// This predates the is_signal flip for every other thread-level literal
+// (CalendarOnly, ProjectId, dates); Importance joined the risk surface when
+// it moved thread-level. The FE and AI toolset only emit AND-shaped
+// importance filters, so no caller hits this today. If this test starts
+// failing with 6, cross-level OR was fixed — delete this test with joy.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../fixtures", scripts("email_dynamic_query"))
+)]
+async fn test_mixed_or_sender_importance_broadens_known_limitation(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
+ let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
+ let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);
+ let limit = 50;
+
+ let filter = Arc::new(Expr::or(
+ Expr::Literal(EmailLiteral::Sender(Email::Complete(
+ EmailStr::parse_from_str("[email protected]")?.into_owned(),
+ ))),
+ Expr::Literal(EmailLiteral::Importance(true)),
+ ));
+ let query = Query::new(None, SimpleSortMethod::UpdatedAt, filter);
+
+ let results =
+ dynamic::dynamic_email_thread_cursor(&pool, &[link_id], limit, &view, query, "", None)
+ .await?;
+
+ let result_ids: std::collections::HashSet<String> =
+ results.iter().map(|r| r.id.to_string()).collect();
+
+ // Broadened: every thread with a surfaceable message (1-8), including
+ // noise threads 6 and 7 that match neither OR branch.
+ assert_eq!(results.len(), 8, "mixed OR currently broadens to all");
+ assert!(result_ids.contains("20000006-0000-0000-0000-000000000006"));
+ assert!(result_ids.contains("20000007-0000-0000-0000-000000000007"));
+
+ Ok(())
+}
+
#[sqlx::test(
migrator = "MACRO_DB_MIGRATIONS",
fixtures(
@@ -786,6 +836,7 @@ async fn test_dynamic_query_with_importance_filter(pool: Pool<Postgres>) -> anyh
async fn test_dynamic_query_importance_true_email_filters_domain_with_address_override(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);
let limit = 50;
@@ -822,6 +873,7 @@ async fn test_dynamic_query_importance_true_email_filters_domain_with_address_ov
async fn test_dynamic_query_importance_true_email_filters_excludes_trashed_messages(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);
let limit = 50;
@@ -854,6 +906,7 @@ async fn test_dynamic_query_importance_true_email_filters_excludes_trashed_messa
async fn test_dynamic_query_importance_false_email_filters_domain_with_address_override(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);
let limit = 50;
@@ -1041,6 +1094,7 @@ async fn test_dynamic_query_project_id_with_inbox_view(pool: Pool<Postgres>) ->
async fn test_dynamic_query_inbox_view_with_importance_false(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::Inbox);
let limit = 50;
@@ -1147,6 +1201,7 @@ async fn test_dynamic_query_thread_id_with_sender_filter(
async fn test_importance_true_includes_drafts_with_depriority_label(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);
let limit = 50;
@@ -1184,6 +1239,7 @@ async fn test_importance_true_includes_drafts_with_depriority_label(
fixtures(path = "../../../../fixtures", scripts("email_dynamic_query"))
)]
async fn test_importance_true_excludes_trashed_drafts(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);
let limit = 50;
@@ -1226,6 +1282,7 @@ async fn test_importance_true_excludes_trashed_drafts(pool: Pool<Postgres>) -> a
fixtures(path = "../../../../fixtures", scripts("email_dynamic_query"))
)]
async fn test_importance_false_excludes_drafts(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);
let limit = 50;
@@ -1253,7 +1310,7 @@ async fn test_importance_false_excludes_drafts(pool: Pool<Postgres>) -> anyhow::
"importance=false should exclude draft thread 3"
);
- // Only non-draft threads with depriority labels should appear
+ // Only non-signal threads with a surfaceable (non-trash) message appear
assert!(
result_ids.contains("20000006-0000-0000-0000-000000000006"),
"Should include thread 6 (CATEGORY_UPDATES, not a draft)"
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/importance_pagination.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/importance_pagination.rs
index 433003f93b..b24e30f1cb 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/importance_pagination.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/importance_pagination.rs
@@ -22,6 +22,7 @@ fn importance_exclude_filter() -> Arc<Expr<EmailLiteral>> {
async fn test_importance_inbox_pagination_fills_page_and_advances_cursor(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link_id = Uuid::parse_str("a1111111-1111-1111-1111-111111111111")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::Inbox);
let limit = 3;
@@ -111,6 +112,7 @@ async fn test_importance_inbox_pagination_fills_page_and_advances_cursor(
async fn test_importance_inbox_underfilled_page_has_no_more(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link_id = Uuid::parse_str("a1111111-1111-1111-1111-111111111111")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::Inbox);
let limit = 50;
@@ -170,6 +172,7 @@ async fn test_importance_inbox_underfilled_page_has_no_more(
async fn test_importance_pagination_multi_inbox_counts_agree(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
+ sync_all_signal_flags(&pool).await?;
let link1 = Uuid::parse_str("a1111111-1111-1111-1111-111111111111")?;
let link2 = Uuid::parse_str("b2222222-2222-2222-2222-222222222222")?;
let view = PreviewView::StandardLabel(PreviewViewStandardLabel::Inbox);
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/mod.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/mod.rs
index e435d32e92..d4497a2450 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/mod.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/mod.rs
@@ -26,3 +26,17 @@ use macro_user_id::user_id::MacroUserIdStr;
use models_pagination::{Cursor, CursorVal, Query, SimpleSortMethod};
use sqlx::{Pool, Postgres};
use uuid::Uuid;
+
+/// Recomputes `is_signal` for every fixture thread via the real sync
+/// function, so importance tests exercise the heuristic → flag → query chain
+/// end-to-end instead of trusting hand-written fixture verdicts.
+async fn sync_all_signal_flags(pool: &Pool<Postgres>) -> anyhow::Result<()> {
+ let thread_ids: Vec<Uuid> = sqlx::query_scalar!("SELECT id FROM email_threads")
+ .fetch_all(pool)
+ .await?;
+ let mut conn = pool.acquire().await?;
+ for id in thread_ids {
+ super::thread::sync_thread_signal_flag(&mut conn, id).await?;
+ }
+ Ok(())
+}
diff --git a/rust/cloud-storage/email_db_client/src/user_history/mod.rs b/rust/cloud-storage/email_db_client/src/user_history/mod.rs
index 33fe590b21..1eabed36ef 100644
--- a/rust/cloud-storage/email_db_client/src/user_history/mod.rs
+++ b/rust/cloud-storage/email_db_client/src/user_history/mod.rs
@@ -85,6 +85,8 @@ pub async fn get_thread_summary_info(
) AS "is_draft!",
t.is_read,
t.inbox_visible,
+ -- Gmail's IMPORTANT label — a different notion from the
+ -- Importance filter / email_threads.is_signal heuristic.
(
SELECT EXISTS (
SELECT 1
diff --git a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/backfill_signal_flags.rs b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/backfill_signal_flags.rs
index 5ca8936625..e709a9dc47 100644
--- a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/backfill_signal_flags.rs
+++ b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/backfill_signal_flags.rs
@@ -17,6 +17,9 @@
//! - `CONCURRENCY`: Number of users processed concurrently (defaults to 1).
//! - `FULL_RECOMPUTE`: When `true`, recomputes both directions instead of
//! the default set-true-only pass (defaults to false).
+//! - `VERIFY`: When `true`, read-only mode — reports threads whose is_signal
+//! disagrees with the heuristic and exits non-zero if any are found.
+//! Mutually exclusive with `FULL_RECOMPUTE`.
mod config;
mod process;
@@ -26,6 +29,16 @@ use futures::stream::{self, StreamExt};
use macro_entrypoint::MacroEntrypoint;
use sqlx::postgres::PgPoolOptions;
+/// What the per-link count means in log lines, per mode.
+fn mode_noun(mode: config::BackfillMode) -> &'static str {
+ match mode {
+ config::BackfillMode::SetTrueOnly => "flagged",
+ // FullRecompute clears first: the count is threads that ended true.
+ config::BackfillMode::FullRecompute => "computed as signal",
+ config::BackfillMode::Verify => "mismatched",
+ }
+}
+
#[tokio::main]
async fn main() -> anyhow::Result<()> {
println!("Loading configuration...");
@@ -53,12 +66,12 @@ async fn main() -> anyhow::Result<()> {
config.concurrency
);
- let full_recompute = config.full_recompute;
+ let mode = config.mode;
let results: Vec<(String, anyhow::Result<u64>)> = stream::iter(macro_ids)
.map(|macro_id| {
let db_pool = db_pool.clone();
async move {
- let result = process::process_macro_id(&db_pool, &macro_id, full_recompute).await;
+ let result = process::process_macro_id(&db_pool, &macro_id, mode).await;
(macro_id, result)
}
})
@@ -66,9 +79,10 @@ async fn main() -> anyhow::Result<()> {
.enumerate()
.map(|(index, (macro_id, result))| {
match &result {
- Ok(flagged) => println!(
- "=== Completed {macro_id} ({}/{total_users}): flagged {flagged} threads ===",
- index + 1
+ Ok(count) => println!(
+ "=== Completed {macro_id} ({}/{total_users}): {count} threads {} ===",
+ index + 1,
+ mode_noun(mode)
),
Err(e) => println!(
"=== Failed {macro_id} ({}/{total_users}): {e:?} ===",
@@ -80,7 +94,7 @@ async fn main() -> anyhow::Result<()> {
.collect()
.await;
- let total_flagged: u64 = results.iter().filter_map(|(_, r)| r.as_ref().ok()).sum();
+ let total: u64 = results.iter().filter_map(|(_, r)| r.as_ref().ok()).sum();
let failures: Vec<&String> = results
.iter()
.filter(|(_, r)| r.is_err())
@@ -88,7 +102,8 @@ async fn main() -> anyhow::Result<()> {
.collect();
println!(
- "\n=== All macro IDs processed: {total_flagged} threads flagged, {} failures ===",
+ "\n=== All macro IDs processed: {total} threads {}, {} failures ===",
+ mode_noun(mode),
failures.len()
);
if !failures.is_empty() {
@@ -102,5 +117,10 @@ async fn main() -> anyhow::Result<()> {
.join(", ")
);
}
+ if mode == config::BackfillMode::Verify && total > 0 {
+ anyhow::bail!(
+ "{total} threads disagree with the heuristic; run with FULL_RECOMPUTE=true to repair"
+ );
+ }
Ok(())
}
diff --git a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/config.rs b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/config.rs
index 4ab048cc6d..ef040501b5 100644
--- a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/config.rs
+++ b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/config.rs
@@ -10,9 +10,20 @@ pub struct Config {
/// Number of users processed concurrently. Defaults to 1, capped at 50
/// to bound DB connection usage.
pub concurrency: usize,
- /// When true, recomputes is_signal in both directions (clears stale true
- /// flags) instead of the default set-true-only pass.
- pub full_recompute: bool,
+ /// What the per-link pass does; see [`BackfillMode`].
+ pub mode: BackfillMode,
+}
+
+/// What the per-link pass does.
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum BackfillMode {
+ /// Flag matching threads true; never clears (default).
+ SetTrueOnly,
+ /// Clear + re-set per link so stale true flags are repaired.
+ FullRecompute,
+ /// Read-only: count threads whose is_signal disagrees with the
+ /// heuristic. No writes.
+ Verify,
}
env_vars! {
@@ -20,6 +31,7 @@ env_vars! {
struct DatabaseUrl;
struct Concurrency;
struct FullRecompute;
+ struct Verify;
}
impl Config {
@@ -38,6 +50,16 @@ impl Config {
Some(v) => v.parse::<bool>().context("FULL_RECOMPUTE is not a bool")?,
None => false,
};
+ let verify = match Verify::new().ok() {
+ Some(v) => v.parse::<bool>().context("VERIFY is not a bool")?,
+ None => false,
+ };
+ let mode = match (verify, full_recompute) {
+ (true, true) => anyhow::bail!("VERIFY and FULL_RECOMPUTE are mutually exclusive"),
+ (true, false) => BackfillMode::Verify,
+ (false, true) => BackfillMode::FullRecompute,
+ (false, false) => BackfillMode::SetTrueOnly,
+ };
Ok(Self {
macro_ids: MacroIds::new().ok().map(|v| v.to_string()),
@@ -45,7 +67,7 @@ impl Config {
.context("DATABASE_URL not set")?
.to_string(),
concurrency,
- full_recompute,
+ mode,
})
}
}
diff --git a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/process.rs b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/process.rs
index bcb6d4a55e..67932e1e99 100644
--- a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/process.rs
+++ b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/process.rs
@@ -1,14 +1,16 @@
+use crate::config::BackfillMode;
use sqlx::types::Uuid;
/// Flags every thread on one of the user's links that has at least one
/// non-TRASH message matching the importance heuristic. One statement per
/// link keeps transactions small and the progress output per-mailbox.
-/// With `full_recompute`, stale true flags are also cleared (clear + re-set
-/// in one transaction per link). Returns the number of threads flagged.
+/// `FullRecompute` also clears stale true flags (clear + re-set in one
+/// transaction per link); `Verify` is read-only and counts disagreements.
+/// Returns the number of threads flagged (or mismatched, for `Verify`).
pub async fn process_macro_id(
pool: &sqlx::PgPool,
macro_id: &str,
- full_recompute: bool,
+ mode: BackfillMode,
) -> anyhow::Result<u64> {
let link_ids: Vec<Uuid> =
sqlx::query_scalar!("SELECT id FROM email_links WHERE macro_id = $1", macro_id)
@@ -20,14 +22,21 @@ pub async fn process_macro_id(
return Ok(0);
}
- let mut total_flagged = 0u64;
+ let mut total = 0u64;
for link_id in link_ids {
+ if mode == BackfillMode::Verify {
+ let mismatched = verify_link(pool, link_id).await?;
+ println!("[{macro_id}] link {link_id}: {mismatched} threads mismatched");
+ total += mismatched;
+ continue;
+ }
+
let mut tx = pool.begin().await?;
// Full recompute: clear everything first so the set-true pass below
// yields exactly the heuristic verdict. Same transaction, so readers
// never observe the intermediate all-false state.
- if full_recompute {
+ if mode == BackfillMode::FullRecompute {
let cleared = sqlx::query!(
r#"
UPDATE email_threads
@@ -147,11 +156,122 @@ pub async fn process_macro_id(
tx.commit().await?;
// Prefix with the user so interleaved concurrent output stays readable.
- println!("[{macro_id}] link {link_id}: flagged {flagged} threads");
- total_flagged += flagged;
+ // FullRecompute clears first, so its count is "threads that ended
+ // true", not rows changed.
+ if mode == BackfillMode::FullRecompute {
+ println!("[{macro_id}] link {link_id}: computed {flagged} signal threads");
+ } else {
+ println!("[{macro_id}] link {link_id}: flagged {flagged} threads");
+ }
+ total += flagged;
}
- Ok(total_flagged)
+ Ok(total)
+}
+
+/// Read-only differential check: counts the link's threads whose stored
+/// is_signal disagrees with the live heuristic verdict. Mirrors
+/// sync_thread_signal_flag (email_db_client/threads/update.rs).
+async fn verify_link(pool: &sqlx::PgPool, link_id: Uuid) -> anyhow::Result<u64> {
+ let mismatched = sqlx::query_scalar!(
+ r#"
+ SELECT count(*) AS "mismatched!"
+ FROM email_threads t
+ WHERE t.link_id = $1
+ AND t.is_signal IS DISTINCT FROM EXISTS (
+ SELECT 1
+ FROM email_messages m
+ WHERE m.thread_id = t.id
+ AND NOT EXISTS (
+ SELECT 1 FROM email_message_labels ml
+ JOIN email_labels l ON ml.label_id = l.id
+ WHERE ml.message_id = m.id AND l.name = 'TRASH'
+ )
+ AND (
+ (
+ EXISTS (
+ SELECT 1
+ FROM email_contacts sender_c
+ JOIN email_filters ef
+ ON ef.link_id = m.link_id
+ AND ef.email_address IS NOT NULL
+ AND LOWER(ef.email_address) = LOWER(sender_c.email_address)
+ WHERE sender_c.id = m.from_contact_id
+ AND ef.is_important = TRUE
+ )
+ OR EXISTS (
+ SELECT 1
+ FROM email_contacts sender_c
+ JOIN email_filters ef
+ ON ef.link_id = m.link_id
+ AND ef.email_domain IS NOT NULL
+ AND LOWER(ef.email_domain) = LOWER(SPLIT_PART(sender_c.email_address, '@', 2))
+ WHERE sender_c.id = m.from_contact_id
+ AND ef.is_important = TRUE
+ AND NOT EXISTS (
+ SELECT 1 FROM email_filters ef_addr
+ WHERE ef_addr.link_id = m.link_id
+ AND ef_addr.email_address IS NOT NULL
+ AND LOWER(ef_addr.email_address) = LOWER(sender_c.email_address)
+ AND ef_addr.is_important = FALSE
+ )
+ )
+ )
+ OR (
+ NOT (
+ EXISTS (
+ SELECT 1
+ FROM email_contacts sender_c
+ JOIN email_filters ef
+ ON ef.link_id = m.link_id
+ AND ef.email_address IS NOT NULL
+ AND LOWER(ef.email_address) = LOWER(sender_c.email_address)
+ WHERE sender_c.id = m.from_contact_id
+ AND ef.is_important = FALSE
+ )
+ OR EXISTS (
+ SELECT 1
+ FROM email_contacts sender_c
+ JOIN email_filters ef
+ ON ef.link_id = m.link_id
+ AND ef.email_domain IS NOT NULL
+ AND LOWER(ef.email_domain) = LOWER(SPLIT_PART(sender_c.email_address, '@', 2))
+ WHERE sender_c.id = m.from_contact_id
+ AND ef.is_important = FALSE
+ AND NOT EXISTS (
+ SELECT 1 FROM email_filters ef_addr
+ WHERE ef_addr.link_id = m.link_id
+ AND ef_addr.email_address IS NOT NULL
+ AND LOWER(ef_addr.email_address) = LOWER(sender_c.email_address)
+ AND ef_addr.is_important = TRUE
+ )
+ )
+ )
+ AND (
+ m.is_draft = TRUE
+ OR EXISTS (
+ SELECT 1 FROM email_message_labels ml
+ JOIN email_labels l ON ml.label_id = l.id
+ WHERE ml.message_id = m.id
+ AND l.name IN ('CATEGORY_PERSONAL', 'SENT', 'DRAFT')
+ )
+ OR NOT EXISTS (
+ SELECT 1 FROM email_message_labels ml
+ JOIN email_labels l ON ml.label_id = l.id
+ WHERE ml.message_id = m.id
+ AND l.name IN ('CATEGORY_UPDATES', 'CATEGORY_PROMOTIONS', 'CATEGORY_SOCIAL', 'CATEGORY_FORUMS')
+ )
+ )
+ )
+ )
+ )
+ "#,
+ link_id
+ )
+ .fetch_one(pool)
+ .await?;
+
+ Ok(mismatched as u64)
}
/// Every macro ID that owns at least one email link. Connected secondary
diff --git a/rust/cloud-storage/item_filters/src/ast/email.rs b/rust/cloud-storage/item_filters/src/ast/email.rs
index 75b141749d..fdbdb9bf49 100644
--- a/rust/cloud-storage/item_filters/src/ast/email.rs
+++ b/rust/cloud-storage/item_filters/src/ast/email.rs
@@ -53,7 +53,9 @@ pub enum EmailLiteral {
Owner(Uuid),
/// This value filters by project ID
ProjectId(String),
- /// This node value filters by email importance. false short-circuits to match nothing.
+ /// Filters by email importance, evaluated against the denormalized
+ /// `email_threads.is_signal` flag: true matches signal threads, false
+ /// matches noise threads.
Importance(bool),
/// This node value filters by notification done state for emails.
NotificationDone(bool),