Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(email): read importance from the denormalized is_signal flag (#4673)
Evan Hutnikcf2fe1b8773a
diff --git a/rust/cloud-storage/.sqlx/query-039dd65419a635365c37d46699bf115e771ceae4526e0a13928152e2bb1bb9b3.json b/rust/cloud-storage/.sqlx/query-039dd65419a635365c37d46699bf115e771ceae4526e0a13928152e2bb1bb9b3.jsondeleted file mode 100644index 887d1272a9..0000000000--- a/rust/cloud-storage/.sqlx/query-039dd65419a635365c37d46699bf115e771ceae4526e0a13928152e2bb1bb9b3.json+++ /dev/null@@ -1,107 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n SELECT\n t.id as thread_id,\n -- Return timestamps from the chosen messages directly\n COALESCE(earliest_msg.sent_at, earliest_msg.updated_at) as \"first_message_ts!\",\n COALESCE(latest_msg.sent_at, latest_msg.updated_at) as \"last_message_ts!\",\n -- Last time user viewed this thread\n uh.updated_at as \"viewed_at?\",\n latest_msg.snippet,\n earliest_msg.subject as \"subject?\",\n l.macro_id,\n t.link_id,\n latest_msg.sender as sender,\n latest_msg.pretty_sender as \"pretty_sender!\",\n latest_msg.trash_label as trash_label,\n (\n SELECT m_latest.is_draft\n FROM email_messages m_latest\n WHERE m_latest.thread_id = t.id\n AND m_latest.link_id = t.link_id\n ORDER BY m_latest.internal_date_ts DESC NULLS LAST\n LIMIT 1\n ) AS \"is_draft!\",\n t.is_read,\n t.inbox_visible,\n (\n SELECT EXISTS (\n SELECT 1\n FROM email_messages m_imp\n JOIN email_message_labels ml ON m_imp.id = ml.message_id\n JOIN email_labels l ON ml.label_id = l.id\n WHERE m_imp.thread_id = t.id\n AND l.link_id = t.link_id\n AND l.name = 'IMPORTANT'\n )\n ) AS \"is_important!\"\n FROM email_threads t\n LEFT JOIN email_user_history uh ON uh.thread_id = t.id AND uh.link_id = t.link_id\n LEFT JOIN email_links l ON l.id = t.link_id\n -- LATERAL join for LATEST message\n -- JOIN (Inner) acts as a filter to ensure we only return threads that actually have messages\n JOIN LATERAL (\n SELECT\n m2.snippet,\n m2.sent_at,\n m2.updated_at,\n c.email_address as sender,\n COALESCE(c.name, c.email_address) as pretty_sender,\n EXISTS(\n SELECT 1 \n FROM email_message_labels eml \n JOIN email_labels el ON el.id = eml.label_id \n WHERE eml.message_id = m2.id \n AND el.provider_label_id = 'TRASH'\n ) as trash_label\n FROM email_messages m2\n LEFT JOIN email_contacts c ON c.id = m2.from_contact_id\n WHERE m2.thread_id = t.id\n AND m2.link_id = t.link_id\n ORDER BY\n (CASE WHEN m2.is_draft = false AND m2.sent_at IS NOT NULL THEN 0 ELSE 1 END) ASC,\n COALESCE(m2.sent_at, m2.updated_at) DESC NULLS LAST\n LIMIT 1\n ) latest_msg ON true\n -- LATERAL join for EARLIEST message\n LEFT JOIN LATERAL (\n SELECT\n m3.subject,\n m3.sent_at,\n m3.updated_at\n FROM email_messages m3\n WHERE m3.thread_id = t.id\n AND m3.link_id = t.link_id\n ORDER BY\n -- 1. Priority: Non-drafts with valid sent_at come first (0), everything else is fallback (1)\n (CASE WHEN m3.is_draft = false AND m3.sent_at IS NOT NULL THEN 0 ELSE 1 END) ASC,\n -- 2. Sort by time (Oldest first)\n COALESCE(m3.sent_at, m3.updated_at) ASC NULLS LAST\n LIMIT 1\n ) earliest_msg ON true\n WHERE t.id = ANY($2)\n AND t.link_id = ANY($1)\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "thread_id",- "type_info": "Uuid"- },- {- "ordinal": 1,- "name": "first_message_ts!",- "type_info": "Timestamptz"- },- {- "ordinal": 2,- "name": "last_message_ts!",- "type_info": "Timestamptz"- },- {- "ordinal": 3,- "name": "viewed_at?",- "type_info": "Timestamptz"- },- {- "ordinal": 4,- "name": "snippet",- "type_info": "Text"- },- {- "ordinal": 5,- "name": "subject?",- "type_info": "Text"- },- {- "ordinal": 6,- "name": "macro_id",- "type_info": "Text"- },- {- "ordinal": 7,- "name": "link_id",- "type_info": "Uuid"- },- {- "ordinal": 8,- "name": "sender",- "type_info": "Varchar"- },- {- "ordinal": 9,- "name": "pretty_sender!",- "type_info": "Varchar"- },- {- "ordinal": 10,- "name": "trash_label",- "type_info": "Bool"- },- {- "ordinal": 11,- "name": "is_draft!",- "type_info": "Bool"- },- {- "ordinal": 12,- "name": "is_read",- "type_info": "Bool"- },- {- "ordinal": 13,- "name": "inbox_visible",- "type_info": "Bool"- },- {- "ordinal": 14,- "name": "is_important!",- "type_info": "Bool"- }- ],- "parameters": {- "Left": [- "UuidArray",- "UuidArray"- ]- },- "nullable": [- false,- null,- null,- false,- true,- true,- false,- false,- false,- null,- null,- null,- false,- false,- null- ]- },- "hash": "039dd65419a635365c37d46699bf115e771ceae4526e0a13928152e2bb1bb9b3"-}diff --git a/rust/cloud-storage/.sqlx/query-3b9b441b224f60a4832dee29e4151a2bc6a87fe87544bf4fdfeee20aeec7c5bf.json b/rust/cloud-storage/.sqlx/query-3b9b441b224f60a4832dee29e4151a2bc6a87fe87544bf4fdfeee20aeec7c5bf.jsonnew file mode 100644index 0000000000..3a34936eb7--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-3b9b441b224f60a4832dee29e4151a2bc6a87fe87544bf4fdfeee20aeec7c5bf.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT count(*) AS \"mismatched!\"\n FROM email_threads t\n WHERE t.link_id = $1\n AND t.is_signal IS DISTINCT FROM EXISTS (\n SELECT 1\n FROM email_messages m\n WHERE m.thread_id = t.id\n AND NOT EXISTS (\n SELECT 1 FROM email_message_labels ml\n JOIN email_labels l ON ml.label_id = l.id\n WHERE ml.message_id = m.id AND l.name = 'TRASH'\n )\n AND (\n (\n EXISTS (\n SELECT 1\n FROM email_contacts sender_c\n JOIN email_filters ef\n ON ef.link_id = m.link_id\n AND ef.email_address IS NOT NULL\n AND LOWER(ef.email_address) = LOWER(sender_c.email_address)\n WHERE sender_c.id = m.from_contact_id\n AND ef.is_important = TRUE\n )\n OR EXISTS (\n SELECT 1\n FROM email_contacts sender_c\n JOIN email_filters ef\n ON ef.link_id = m.link_id\n AND ef.email_domain IS NOT NULL\n AND LOWER(ef.email_domain) = LOWER(SPLIT_PART(sender_c.email_address, '@', 2))\n WHERE sender_c.id = m.from_contact_id\n AND ef.is_important = TRUE\n AND NOT EXISTS (\n SELECT 1 FROM email_filters ef_addr\n WHERE ef_addr.link_id = m.link_id\n AND ef_addr.email_address IS NOT NULL\n AND LOWER(ef_addr.email_address) = LOWER(sender_c.email_address)\n AND ef_addr.is_important = FALSE\n )\n )\n )\n OR (\n NOT (\n EXISTS (\n SELECT 1\n FROM email_contacts sender_c\n JOIN email_filters ef\n ON ef.link_id = m.link_id\n AND ef.email_address IS NOT NULL\n AND LOWER(ef.email_address) = LOWER(sender_c.email_address)\n WHERE sender_c.id = m.from_contact_id\n AND ef.is_important = FALSE\n )\n OR EXISTS (\n SELECT 1\n FROM email_contacts sender_c\n JOIN email_filters ef\n ON ef.link_id = m.link_id\n AND ef.email_domain IS NOT NULL\n AND LOWER(ef.email_domain) = LOWER(SPLIT_PART(sender_c.email_address, '@', 2))\n WHERE sender_c.id = m.from_contact_id\n AND ef.is_important = FALSE\n AND NOT EXISTS (\n SELECT 1 FROM email_filters ef_addr\n WHERE ef_addr.link_id = m.link_id\n AND ef_addr.email_address IS NOT NULL\n AND LOWER(ef_addr.email_address) = LOWER(sender_c.email_address)\n AND ef_addr.is_important = TRUE\n )\n )\n )\n AND (\n m.is_draft = TRUE\n OR EXISTS (\n SELECT 1 FROM email_message_labels ml\n JOIN email_labels l ON ml.label_id = l.id\n WHERE ml.message_id = m.id\n AND l.name IN ('CATEGORY_PERSONAL', 'SENT', 'DRAFT')\n )\n OR NOT EXISTS (\n SELECT 1 FROM email_message_labels ml\n JOIN email_labels l ON ml.label_id = l.id\n WHERE ml.message_id = m.id\n AND l.name IN ('CATEGORY_UPDATES', 'CATEGORY_PROMOTIONS', 'CATEGORY_SOCIAL', 'CATEGORY_FORUMS')\n )\n )\n )\n )\n )\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "mismatched!",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "3b9b441b224f60a4832dee29e4151a2bc6a87fe87544bf4fdfeee20aeec7c5bf"+}diff --git a/rust/cloud-storage/.sqlx/query-e3effaf90fa2b1bf1dc615ce0c74c1afc8d0e6774e71862ee0d39b5381647308.json b/rust/cloud-storage/.sqlx/query-e3effaf90fa2b1bf1dc615ce0c74c1afc8d0e6774e71862ee0d39b5381647308.jsonnew file mode 100644index 0000000000..8919fd97da--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-e3effaf90fa2b1bf1dc615ce0c74c1afc8d0e6774e71862ee0d39b5381647308.json@@ -0,0 +1,107 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT\n t.id as thread_id,\n -- Return timestamps from the chosen messages directly\n COALESCE(earliest_msg.sent_at, earliest_msg.updated_at) as \"first_message_ts!\",\n COALESCE(latest_msg.sent_at, latest_msg.updated_at) as \"last_message_ts!\",\n -- Last time user viewed this thread\n uh.updated_at as \"viewed_at?\",\n latest_msg.snippet,\n earliest_msg.subject as \"subject?\",\n l.macro_id,\n t.link_id,\n latest_msg.sender as sender,\n latest_msg.pretty_sender as \"pretty_sender!\",\n latest_msg.trash_label as trash_label,\n (\n SELECT m_latest.is_draft\n FROM email_messages m_latest\n WHERE m_latest.thread_id = t.id\n AND m_latest.link_id = t.link_id\n ORDER BY m_latest.internal_date_ts DESC NULLS LAST\n LIMIT 1\n ) AS \"is_draft!\",\n t.is_read,\n t.inbox_visible,\n -- Gmail's IMPORTANT label — a different notion from the\n -- Importance filter / email_threads.is_signal heuristic.\n (\n SELECT EXISTS (\n SELECT 1\n FROM email_messages m_imp\n JOIN email_message_labels ml ON m_imp.id = ml.message_id\n JOIN email_labels l ON ml.label_id = l.id\n WHERE m_imp.thread_id = t.id\n AND l.link_id = t.link_id\n AND l.name = 'IMPORTANT'\n )\n ) AS \"is_important!\"\n FROM email_threads t\n LEFT JOIN email_user_history uh ON uh.thread_id = t.id AND uh.link_id = t.link_id\n LEFT JOIN email_links l ON l.id = t.link_id\n -- LATERAL join for LATEST message\n -- JOIN (Inner) acts as a filter to ensure we only return threads that actually have messages\n JOIN LATERAL (\n SELECT\n m2.snippet,\n m2.sent_at,\n m2.updated_at,\n c.email_address as sender,\n COALESCE(c.name, c.email_address) as pretty_sender,\n EXISTS(\n SELECT 1 \n FROM email_message_labels eml \n JOIN email_labels el ON el.id = eml.label_id \n WHERE eml.message_id = m2.id \n AND el.provider_label_id = 'TRASH'\n ) as trash_label\n FROM email_messages m2\n LEFT JOIN email_contacts c ON c.id = m2.from_contact_id\n WHERE m2.thread_id = t.id\n AND m2.link_id = t.link_id\n ORDER BY\n (CASE WHEN m2.is_draft = false AND m2.sent_at IS NOT NULL THEN 0 ELSE 1 END) ASC,\n COALESCE(m2.sent_at, m2.updated_at) DESC NULLS LAST\n LIMIT 1\n ) latest_msg ON true\n -- LATERAL join for EARLIEST message\n LEFT JOIN LATERAL (\n SELECT\n m3.subject,\n m3.sent_at,\n m3.updated_at\n FROM email_messages m3\n WHERE m3.thread_id = t.id\n AND m3.link_id = t.link_id\n ORDER BY\n -- 1. Priority: Non-drafts with valid sent_at come first (0), everything else is fallback (1)\n (CASE WHEN m3.is_draft = false AND m3.sent_at IS NOT NULL THEN 0 ELSE 1 END) ASC,\n -- 2. Sort by time (Oldest first)\n COALESCE(m3.sent_at, m3.updated_at) ASC NULLS LAST\n LIMIT 1\n ) earliest_msg ON true\n WHERE t.id = ANY($2)\n AND t.link_id = ANY($1)\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "thread_id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "first_message_ts!",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 2,+ "name": "last_message_ts!",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 3,+ "name": "viewed_at?",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 4,+ "name": "snippet",+ "type_info": "Text"+ },+ {+ "ordinal": 5,+ "name": "subject?",+ "type_info": "Text"+ },+ {+ "ordinal": 6,+ "name": "macro_id",+ "type_info": "Text"+ },+ {+ "ordinal": 7,+ "name": "link_id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 8,+ "name": "sender",+ "type_info": "Varchar"+ },+ {+ "ordinal": 9,+ "name": "pretty_sender!",+ "type_info": "Varchar"+ },+ {+ "ordinal": 10,+ "name": "trash_label",+ "type_info": "Bool"+ },+ {+ "ordinal": 11,+ "name": "is_draft!",+ "type_info": "Bool"+ },+ {+ "ordinal": 12,+ "name": "is_read",+ "type_info": "Bool"+ },+ {+ "ordinal": 13,+ "name": "inbox_visible",+ "type_info": "Bool"+ },+ {+ "ordinal": 14,+ "name": "is_important!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "UuidArray",+ "UuidArray"+ ]+ },+ "nullable": [+ false,+ null,+ null,+ false,+ true,+ true,+ false,+ false,+ false,+ null,+ null,+ null,+ false,+ false,+ null+ ]+ },+ "hash": "e3effaf90fa2b1bf1dc615ce0c74c1afc8d0e6774e71862ee0d39b5381647308"+}diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rsindex 437cdd51d8..9c00b1a369 100644--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs@@ -35,6 +35,7 @@ pub(super) fn has_thread_literals(ast: &Expr<EmailLiteral>) -> bool {| EmailLiteral::Owner(_)| EmailLiteral::ProjectId(_)| EmailLiteral::CalendarOnly(_)+ | EmailLiteral::Importance(_)| EmailLiteral::CreatedAt(_)| EmailLiteral::UpdatedAt(_)| EmailLiteral::Property(_),@@ -54,6 +55,7 @@ pub(super) fn has_message_literals(ast: &Expr<EmailLiteral>) -> bool {| EmailLiteral::ProjectId(_)| EmailLiteral::Shared(_)| EmailLiteral::CalendarOnly(_)+ | EmailLiteral::Importance(_)| EmailLiteral::CreatedAt(_)| EmailLiteral::UpdatedAt(_)| EmailLiteral::Property(_),@@ -62,15 +64,6 @@ pub(super) fn has_message_literals(ast: &Expr<EmailLiteral>) -> bool {})}-pub(super) fn has_importance_literal(ast: &Expr<EmailLiteral>) -> bool {- ast.collapse_frames(|frame| match frame {- filter_ast::ExprFrame::And(a, b) | filter_ast::ExprFrame::Or(a, b) => a || b,- filter_ast::ExprFrame::Not(a) => a,- filter_ast::ExprFrame::Literal(EmailLiteral::Importance(_)) => true,- filter_ast::ExprFrame::Literal(_) => false,- })-}-#[derive(Clone, Copy)]enum AddressKind {Sender,@@ -184,44 +177,6 @@ fn build_address_text_match(}}-fn build_sender_importance_override_filter(is_important: bool) -> SqlFragment {- let importance_literal = if is_important { "TRUE" } else { "FALSE" };- let opposite_importance_literal = if is_important { "FALSE" } else { "TRUE" };-- SqlFragment::raw(format!(- r#"(- EXISTS (- SELECT 1- FROM email_contacts sender_c- JOIN email_filters ef- ON ef.link_id = m.link_id- AND ef.email_address IS NOT NULL- AND LOWER(ef.email_address) = LOWER(sender_c.email_address)- WHERE sender_c.id = m.from_contact_id- AND ef.is_important = {importance_literal}- )- OR EXISTS (- SELECT 1- FROM email_contacts sender_c- JOIN email_filters ef- ON ef.link_id = m.link_id- AND ef.email_domain IS NOT NULL- AND LOWER(ef.email_domain) = LOWER(split_part(sender_c.email_address, '@', 2))- WHERE sender_c.id = m.from_contact_id- AND ef.is_important = {importance_literal}- AND NOT EXISTS (- SELECT 1- FROM email_filters ef_addr- WHERE ef_addr.link_id = m.link_id- AND ef_addr.email_address IS NOT NULL- AND LOWER(ef_addr.email_address) = LOWER(sender_c.email_address)- AND ef_addr.is_important = {opposite_importance_literal}- )- )- )"#,- ))-}-pub(super) fn build_message_email_filter(ast: &Expr<EmailLiteral>,resolved: &ResolvedFilters,@@ -232,9 +187,7 @@ pub(super) fn build_message_email_filter(filter_ast::ExprFrame::Not(a) => SqlFragment::not(a),filter_ast::ExprFrame::Literal(- EmailLiteral::ThreadId(_)- | EmailLiteral::Owner(_)- | EmailLiteral::ProjectId(_),+ EmailLiteral::ThreadId(_) | EmailLiteral::Owner(_) | EmailLiteral::ProjectId(_),) => SqlFragment::raw("TRUE"),filter_ast::ExprFrame::Literal(EmailLiteral::Sender(email)) => {@@ -253,79 +206,7 @@ pub(super) fn build_message_email_filter(build_address_predicate_on_m(AddressKind::Bcc, &email, resolved)}- filter_ast::ExprFrame::Literal(EmailLiteral::Importance(true)) => {- let mut f = SqlFragment::raw(- r#"(- NOT EXISTS (- SELECT 1 FROM email_message_labels ml- JOIN email_labels l ON ml.label_id = l.id- WHERE ml.message_id = m.id- AND l.name = 'TRASH'- )- AND (- "#,- );- f.extend(build_sender_importance_override_filter(true));- f.push_raw(- r#"- OR (- NOT "#,- );- f.extend(build_sender_importance_override_filter(false));- f.push_raw(- r#"- AND (- m.is_draft = TRUE- OR EXISTS (- SELECT 1 FROM email_message_labels ml- JOIN email_labels l ON ml.label_id = l.id- WHERE ml.message_id = m.id- AND l.name IN ('CATEGORY_PERSONAL', 'SENT', 'DRAFT')- )- OR NOT EXISTS (- SELECT 1 FROM email_message_labels ml- JOIN email_labels l ON ml.label_id = l.id- WHERE ml.message_id = m.id- AND l.name IN ('CATEGORY_UPDATES', 'CATEGORY_PROMOTIONS', 'CATEGORY_SOCIAL', 'CATEGORY_FORUMS')- )- )- )- )- )"#,- );- f- }- filter_ast::ExprFrame::Literal(EmailLiteral::Importance(false)) => {- let mut f = SqlFragment::raw(- r#"(- "#,- );- f.extend(build_sender_importance_override_filter(false));- f.push_raw(- r#"- OR (- NOT "#,- );- f.extend(build_sender_importance_override_filter(true));- f.push_raw(- r#"- AND NOT EXISTS (- SELECT 1 FROM email_message_labels ml- JOIN email_labels l ON ml.label_id = l.id- WHERE ml.message_id = m.id- AND l.name IN ('CATEGORY_PERSONAL', 'SENT', 'DRAFT')- )- AND EXISTS (- SELECT 1 FROM email_message_labels ml- JOIN email_labels l ON ml.label_id = l.id- WHERE ml.message_id = m.id- AND l.name IN ('CATEGORY_UPDATES', 'CATEGORY_PROMOTIONS', 'CATEGORY_SOCIAL', 'CATEGORY_FORUMS')- )- )- )"#,- );- f- }+ filter_ast::ExprFrame::Literal(EmailLiteral::Importance(_)) => SqlFragment::raw("TRUE"),filter_ast::ExprFrame::Literal(EmailLiteral::NotificationDone(_)) => {SqlFragment::raw("TRUE")}@@ -360,7 +241,7 @@ pub(super) fn has_address_literals(ast: &Expr<EmailLiteral>) -> bool {/// True if the subtree contains only address literals (Sender/Cc/Bcc/Recipient)/// composed via And/Or/Not. Used to decide whether a top-level conjunct can be/// safely pushed into the candidate-thread pre-filter without risking false-/// negatives (e.g., `Sender(X) OR Importance(true)` cannot be reduced to just+/// negatives (e.g., `Sender(X) OR CalendarOnly(true)` cannot be reduced to just/// `Sender(X)` at the candidate stage).fn is_pure_address_subtree(expr: &Expr<EmailLiteral>) -> bool {expr.collapse_frames(|frame| match frame {@@ -473,13 +354,12 @@ pub(super) fn build_thread_address_filter(ast: &Expr<EmailLiteral>) -> SqlFragme/// True when the candidate WHERE must mirror the CROSS JOIN LATERAL's/// message match via a correlated EXISTS rather than the address-only/// `matching_threads` CTE. That's the case when the lateral applies a-/// per-message filter the address CTE doesn't model — an `Importance`-/// literal, or a non-empty view-level message filter (Starred / Drafts /-/// Important / …). Without this, the candidate `LIMIT` counts threads that-/// the lateral later drops, so the page under-fills while still emitting a-/// cursor.-pub(super) fn wants_message_exists_pushdown(ast: &Expr<EmailLiteral>, view: &PreviewView) -> bool {- has_importance_literal(ast) || !build_view_message_filter(view).is_empty()+/// per-message filter the address CTE doesn't model — a non-empty+/// view-level message filter (Starred / Drafts / Important / …). Without+/// this, the candidate `LIMIT` counts threads that the lateral later+/// drops, so the page under-fills while still emitting a cursor.+pub(super) fn wants_message_exists_pushdown(view: &PreviewView) -> bool {+ !build_view_message_filter(view).is_empty()}/// Builds the candidate-stage mirror of the lateral message match:@@ -775,6 +655,16 @@ pub(super) fn build_thread_email_filter(SqlFragment::raw("TRUE")}+ // Denormalized importance flag maintained by update_thread_metadata+ // (sync_thread_signal_flag) and the email_filters resync fan-out.+ filter_ast::ExprFrame::Literal(EmailLiteral::Importance(true)) => {+ SqlFragment::raw("t.is_signal")+ }++ filter_ast::ExprFrame::Literal(EmailLiteral::Importance(false)) => {+ SqlFragment::raw("(NOT t.is_signal)")+ }+filter_ast::ExprFrame::Literal(EmailLiteral::CreatedAt(ref lit)) => {date_predicate("t.created_at", lit)}@@ -792,7 +682,6 @@ pub(super) fn build_thread_email_filter(| EmailLiteral::Cc(_)| EmailLiteral::Bcc(_)| EmailLiteral::Recipient(_)- | EmailLiteral::Importance(_)| EmailLiteral::NotificationDone(_)| EmailLiteral::NotificationSeen(_)| EmailLiteral::Shared(_),diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rsindex 5b9f7ae26c..23a07da1a5 100644--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs@@ -304,12 +304,12 @@ fn push_thread_candidate_select(}// Ensure the candidate LIMIT only counts threads that will survive the- // CROSS JOIN LATERAL's message match. When a per-message filter is in- // play (importance, or a view-level message filter) mirror the full- // lateral predicate as a correlated EXISTS; otherwise push address-only- // constraints through the index-driven `matching_threads` CTE referenced- // via `t.id IN (SELECT thread_id FROM matching_threads)`.- if wants_message_exists_pushdown(email_filter, view) {+ // CROSS JOIN LATERAL's message match. When a view-level message filter+ // is in play mirror the full lateral predicate as a correlated EXISTS;+ // otherwise push address-only constraints through the index-driven+ // `matching_threads` CTE referenced via+ // `t.id IN (SELECT thread_id FROM matching_threads)`.+ if wants_message_exists_pushdown(view) {build_thread_message_exists_filter(email_filter, view, ¶ms.resolved).push_into(builder);} else if has_address_literals(email_filter) {build_thread_address_filter(email_filter).push_into(builder);@@ -406,11 +406,10 @@ fn build_query(let defer_uh = !sort_uses_view_history(¶ms.sort_method_str);let needs_shared_cte = !matches!(params.shared, SharedEmailFilter::Exclude);- // When the candidate stage pushes a full per-message EXISTS (importance,- // or a view-level message filter), it already enforces "thread has a- // message the lateral will surface", so the address-only- // `matching_threads` CTE is redundant.- let matching_threads_body = if wants_message_exists_pushdown(email_filter, view) {+ // When the candidate stage pushes a full per-message EXISTS (view-level+ // message filter), it already enforces "thread has a message the lateral+ // will surface", so the address-only `matching_threads` CTE is redundant.+ let matching_threads_body = if wants_message_exists_pushdown(view) {None} else {// Owned-only, non-team queries can scope the CTE's contact/message@@ -464,6 +463,9 @@ fn build_query(builder.push("t.viewed_at,");}+ // The is_important output column reflects Gmail's IMPORTANT label — a+ // different notion from the Importance filter, which reads the+ // denormalized email_threads.is_signal heuristic.builder.push(r#"t.project_id,diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rsindex 24466d7bd3..f77c394f95 100644--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs@@ -254,49 +254,56 @@ fn test_partial_and_domain_emit_different_predicates_for_same_string() {}#[test]-fn test_build_message_email_filter_importance_true_includes_drafts() {- let expr = Expr::Literal(EmailLiteral::Importance(true));- let result = build_message_email_filter(&expr, &ResolvedFilters::empty());- let debug = result.to_debug_sql();-- assert!(debug.contains("m.is_draft = TRUE"));+fn test_importance_compiles_to_thread_signal_flag() {+ let thread = build_thread_email_filter(+ &Expr::Literal(EmailLiteral::Importance(true)),+ "t.updated_at",+ )+ .to_debug_sql();+ assert!(thread.contains("t.is_signal"));+ assert!(!thread.contains("NOT t.is_signal"));++ let thread = build_thread_email_filter(+ &Expr::Literal(EmailLiteral::Importance(false)),+ "t.updated_at",+ )+ .to_debug_sql();+ assert!(thread.contains("NOT t.is_signal"));+}++#[test]+fn test_importance_is_noop_in_message_filter() {+ // The heuristic lives in the denormalized flag now; the lateral must not+ // re-evaluate it per message.+ for imp in [true, false] {+ let result = build_message_email_filter(+ &Expr::Literal(EmailLiteral::Importance(imp)),+ &ResolvedFilters::empty(),+ );+ let debug = result.to_debug_sql();+ assert!(!debug.contains("email_filters"));+ assert!(!debug.contains("CATEGORY"));+ assert!(!debug.contains("is_important"));+ }}#[test]-fn test_build_message_email_filter_importance_true_excludes_trash() {- let expr = Expr::Literal(EmailLiteral::Importance(true));- let result = build_message_email_filter(&expr, &ResolvedFilters::empty());- let debug = result.to_debug_sql();-- assert!(debug.contains("l.name = 'TRASH'"));- assert!(debug.contains("NOT EXISTS"));-}+fn test_importance_query_has_no_message_exists_pushdown() {+ use item_filters::SharedEmailFilter;-#[test]-fn test_build_message_email_filter_importance_true_includes_email_filters() {- let expr = Expr::Literal(EmailLiteral::Importance(true));- let result = build_message_email_filter(&expr, &ResolvedFilters::empty());- let debug = result.to_debug_sql();-- assert!(debug.contains("FROM email_filters ef"));- assert!(- debug- .contains("LOWER(ef.email_domain) = LOWER(split_part(sender_c.email_address, '@', 2))")+ // The Signal-tab shape: inbox view + Importance(true) + Shared(exclude).+ // The candidate stage must filter on t.is_signal directly, with no+ // per-message EXISTS mirror (the inbox view has no message filter).+ let view = PreviewView::StandardLabel(PreviewViewStandardLabel::Inbox);+ let expr = Expr::and(+ Expr::Literal(EmailLiteral::Importance(true)),+ Expr::Literal(EmailLiteral::Shared(SharedEmailFilter::Exclude)),);- assert!(debug.contains("ef.is_important = TRUE"));- assert!(debug.contains("ef_addr.is_important = FALSE"));-}--#[test]-fn test_build_message_email_filter_importance_false_includes_email_filters() {- let expr = Expr::Literal(EmailLiteral::Importance(false));- let result = build_message_email_filter(&expr, &ResolvedFilters::empty());- let debug = result.to_debug_sql();+ let sql = super::query::debug_build_query_sql(&view, &expr);- assert!(debug.contains("FROM email_filters ef"));- assert!(debug.contains("LOWER(ef.email_address) = LOWER(sender_c.email_address)"));- assert!(debug.contains("ef.is_important = FALSE"));- assert!(debug.contains("ef_addr.is_important = TRUE"));+ assert!(sql.contains("t.is_signal"));+ assert!(!sql.contains("AND EXISTS (SELECT 1 FROM email_messages m WHERE m.thread_id = t.id"));+ assert!(!sql.contains("is_important = "));}#[test]diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/dynamic_query.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/dynamic_query.rsindex 31e1c6c72e..9d1e0b135c 100644--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/dynamic_query.rs+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/dynamic_query.rs@@ -680,6 +680,7 @@ async fn test_dynamic_query_pagination(pool: Pool<Postgres>) -> anyhow::Result<(fixtures(path = "../../../../fixtures", scripts("email_dynamic_query")))]async fn test_dynamic_query_with_importance_filter(pool: Pool<Postgres>) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);let limit = 50;@@ -748,8 +749,10 @@ async fn test_dynamic_query_with_importance_filter(pool: Pool<Postgres>) -> anyhdynamic::dynamic_email_thread_cursor(&pool, &[link_id], limit, &view, query, "", None).await?;- // Threads 6 (CATEGORY_UPDATES) and 7 (CATEGORY_PROMOTIONS)- // Thread 8 has CATEGORY_UPDATES but is excluded because DRAFT is a priority label+ // Threads 6 (CATEGORY_UPDATES) and 7 (CATEGORY_PROMOTIONS). The+ // all-trash threads 9-11 are is_signal=false too, but the lateral's+ // TRASH exclusion leaves them no message to surface, so they still+ // drop out. Thread 8 is excluded because DRAFT made it signal.assert_eq!(results.len(),2,@@ -776,6 +779,53 @@ async fn test_dynamic_query_with_importance_filter(pool: Pool<Postgres>) -> anyhOk(())}+// KNOWN LIMITATION (pinned, not desired): an OR mixing a message-level+// literal (Sender) with a thread-level literal (Importance) broadens to+// match-everything. The thread stage maps Sender to TRUE and the lateral+// maps Importance to TRUE, so `(TRUE OR t.is_signal)` / `(sender OR TRUE)`+// both collapse. The correct result here would be the 6 signal threads+// (john's threads 1/2/5 are all signal), but noise threads 6 and 7 leak in.+// This predates the is_signal flip for every other thread-level literal+// (CalendarOnly, ProjectId, dates); Importance joined the risk surface when+// it moved thread-level. The FE and AI toolset only emit AND-shaped+// importance filters, so no caller hits this today. If this test starts+// failing with 6, cross-level OR was fixed — delete this test with joy.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("email_dynamic_query"))+)]+async fn test_mixed_or_sender_importance_broadens_known_limitation(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;+ let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;+ let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);+ let limit = 50;++ let filter = Arc::new(Expr::or(+ Expr::Literal(EmailLiteral::Sender(Email::Complete(+ EmailStr::parse_from_str("[email protected]")?.into_owned(),+ ))),+ Expr::Literal(EmailLiteral::Importance(true)),+ ));+ let query = Query::new(None, SimpleSortMethod::UpdatedAt, filter);++ let results =+ dynamic::dynamic_email_thread_cursor(&pool, &[link_id], limit, &view, query, "", None)+ .await?;++ let result_ids: std::collections::HashSet<String> =+ results.iter().map(|r| r.id.to_string()).collect();++ // Broadened: every thread with a surfaceable message (1-8), including+ // noise threads 6 and 7 that match neither OR branch.+ assert_eq!(results.len(), 8, "mixed OR currently broadens to all");+ assert!(result_ids.contains("20000006-0000-0000-0000-000000000006"));+ assert!(result_ids.contains("20000007-0000-0000-0000-000000000007"));++ Ok(())+}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS",fixtures(@@ -786,6 +836,7 @@ async fn test_dynamic_query_with_importance_filter(pool: Pool<Postgres>) -> anyhasync fn test_dynamic_query_importance_true_email_filters_domain_with_address_override(pool: Pool<Postgres>,) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);let limit = 50;@@ -822,6 +873,7 @@ async fn test_dynamic_query_importance_true_email_filters_domain_with_address_ovasync fn test_dynamic_query_importance_true_email_filters_excludes_trashed_messages(pool: Pool<Postgres>,) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);let limit = 50;@@ -854,6 +906,7 @@ async fn test_dynamic_query_importance_true_email_filters_excludes_trashed_messaasync fn test_dynamic_query_importance_false_email_filters_domain_with_address_override(pool: Pool<Postgres>,) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);let limit = 50;@@ -1041,6 +1094,7 @@ async fn test_dynamic_query_project_id_with_inbox_view(pool: Pool<Postgres>) ->async fn test_dynamic_query_inbox_view_with_importance_false(pool: Pool<Postgres>,) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::Inbox);let limit = 50;@@ -1147,6 +1201,7 @@ async fn test_dynamic_query_thread_id_with_sender_filter(async fn test_importance_true_includes_drafts_with_depriority_label(pool: Pool<Postgres>,) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);let limit = 50;@@ -1184,6 +1239,7 @@ async fn test_importance_true_includes_drafts_with_depriority_label(fixtures(path = "../../../../fixtures", scripts("email_dynamic_query")))]async fn test_importance_true_excludes_trashed_drafts(pool: Pool<Postgres>) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);let limit = 50;@@ -1226,6 +1282,7 @@ async fn test_importance_true_excludes_trashed_drafts(pool: Pool<Postgres>) -> afixtures(path = "../../../../fixtures", scripts("email_dynamic_query")))]async fn test_importance_false_excludes_drafts(pool: Pool<Postgres>) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link_id = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::All);let limit = 50;@@ -1253,7 +1310,7 @@ async fn test_importance_false_excludes_drafts(pool: Pool<Postgres>) -> anyhow::"importance=false should exclude draft thread 3");- // Only non-draft threads with depriority labels should appear+ // Only non-signal threads with a surfaceable (non-trash) message appearassert!(result_ids.contains("20000006-0000-0000-0000-000000000006"),"Should include thread 6 (CATEGORY_UPDATES, not a draft)"diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/importance_pagination.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/importance_pagination.rsindex 433003f93b..b24e30f1cb 100644--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/importance_pagination.rs+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/importance_pagination.rs@@ -22,6 +22,7 @@ fn importance_exclude_filter() -> Arc<Expr<EmailLiteral>> {async fn test_importance_inbox_pagination_fills_page_and_advances_cursor(pool: Pool<Postgres>,) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link_id = Uuid::parse_str("a1111111-1111-1111-1111-111111111111")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::Inbox);let limit = 3;@@ -111,6 +112,7 @@ async fn test_importance_inbox_pagination_fills_page_and_advances_cursor(async fn test_importance_inbox_underfilled_page_has_no_more(pool: Pool<Postgres>,) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link_id = Uuid::parse_str("a1111111-1111-1111-1111-111111111111")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::Inbox);let limit = 50;@@ -170,6 +172,7 @@ async fn test_importance_inbox_underfilled_page_has_no_more(async fn test_importance_pagination_multi_inbox_counts_agree(pool: Pool<Postgres>,) -> anyhow::Result<()> {+ sync_all_signal_flags(&pool).await?;let link1 = Uuid::parse_str("a1111111-1111-1111-1111-111111111111")?;let link2 = Uuid::parse_str("b2222222-2222-2222-2222-222222222222")?;let view = PreviewView::StandardLabel(PreviewViewStandardLabel::Inbox);diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/mod.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/mod.rsindex e435d32e92..d4497a2450 100644--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/test/mod.rs+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/test/mod.rs@@ -26,3 +26,17 @@ use macro_user_id::user_id::MacroUserIdStr;use models_pagination::{Cursor, CursorVal, Query, SimpleSortMethod};use sqlx::{Pool, Postgres};use uuid::Uuid;++/// Recomputes `is_signal` for every fixture thread via the real sync+/// function, so importance tests exercise the heuristic → flag → query chain+/// end-to-end instead of trusting hand-written fixture verdicts.+async fn sync_all_signal_flags(pool: &Pool<Postgres>) -> anyhow::Result<()> {+ let thread_ids: Vec<Uuid> = sqlx::query_scalar!("SELECT id FROM email_threads")+ .fetch_all(pool)+ .await?;+ let mut conn = pool.acquire().await?;+ for id in thread_ids {+ super::thread::sync_thread_signal_flag(&mut conn, id).await?;+ }+ Ok(())+}diff --git a/rust/cloud-storage/email_db_client/src/user_history/mod.rs b/rust/cloud-storage/email_db_client/src/user_history/mod.rsindex 33fe590b21..1eabed36ef 100644--- a/rust/cloud-storage/email_db_client/src/user_history/mod.rs+++ b/rust/cloud-storage/email_db_client/src/user_history/mod.rs@@ -85,6 +85,8 @@ pub async fn get_thread_summary_info() AS "is_draft!",t.is_read,t.inbox_visible,+ -- Gmail's IMPORTANT label — a different notion from the+ -- Importance filter / email_threads.is_signal heuristic.(SELECT EXISTS (SELECT 1diff --git a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/backfill_signal_flags.rs b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/backfill_signal_flags.rsindex 5ca8936625..e709a9dc47 100644--- a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/backfill_signal_flags.rs+++ b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/backfill_signal_flags.rs@@ -17,6 +17,9 @@//! - `CONCURRENCY`: Number of users processed concurrently (defaults to 1).//! - `FULL_RECOMPUTE`: When `true`, recomputes both directions instead of//! the default set-true-only pass (defaults to false).+//! - `VERIFY`: When `true`, read-only mode — reports threads whose is_signal+//! disagrees with the heuristic and exits non-zero if any are found.+//! Mutually exclusive with `FULL_RECOMPUTE`.mod config;mod process;@@ -26,6 +29,16 @@ use futures::stream::{self, StreamExt};use macro_entrypoint::MacroEntrypoint;use sqlx::postgres::PgPoolOptions;+/// What the per-link count means in log lines, per mode.+fn mode_noun(mode: config::BackfillMode) -> &'static str {+ match mode {+ config::BackfillMode::SetTrueOnly => "flagged",+ // FullRecompute clears first: the count is threads that ended true.+ config::BackfillMode::FullRecompute => "computed as signal",+ config::BackfillMode::Verify => "mismatched",+ }+}+#[tokio::main]async fn main() -> anyhow::Result<()> {println!("Loading configuration...");@@ -53,12 +66,12 @@ async fn main() -> anyhow::Result<()> {config.concurrency);- let full_recompute = config.full_recompute;+ let mode = config.mode;let results: Vec<(String, anyhow::Result<u64>)> = stream::iter(macro_ids).map(|macro_id| {let db_pool = db_pool.clone();async move {- let result = process::process_macro_id(&db_pool, ¯o_id, full_recompute).await;+ let result = process::process_macro_id(&db_pool, ¯o_id, mode).await;(macro_id, result)}})@@ -66,9 +79,10 @@ async fn main() -> anyhow::Result<()> {.enumerate().map(|(index, (macro_id, result))| {match &result {- Ok(flagged) => println!(- "=== Completed {macro_id} ({}/{total_users}): flagged {flagged} threads ===",- index + 1+ Ok(count) => println!(+ "=== Completed {macro_id} ({}/{total_users}): {count} threads {} ===",+ index + 1,+ mode_noun(mode)),Err(e) => println!("=== Failed {macro_id} ({}/{total_users}): {e:?} ===",@@ -80,7 +94,7 @@ async fn main() -> anyhow::Result<()> {.collect().await;- let total_flagged: u64 = results.iter().filter_map(|(_, r)| r.as_ref().ok()).sum();+ let total: u64 = results.iter().filter_map(|(_, r)| r.as_ref().ok()).sum();let failures: Vec<&String> = results.iter().filter(|(_, r)| r.is_err())@@ -88,7 +102,8 @@ async fn main() -> anyhow::Result<()> {.collect();println!(- "\n=== All macro IDs processed: {total_flagged} threads flagged, {} failures ===",+ "\n=== All macro IDs processed: {total} threads {}, {} failures ===",+ mode_noun(mode),failures.len());if !failures.is_empty() {@@ -102,5 +117,10 @@ async fn main() -> anyhow::Result<()> {.join(", "));}+ if mode == config::BackfillMode::Verify && total > 0 {+ anyhow::bail!(+ "{total} threads disagree with the heuristic; run with FULL_RECOMPUTE=true to repair"+ );+ }Ok(())}diff --git a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/config.rs b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/config.rsindex 4ab048cc6d..ef040501b5 100644--- a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/config.rs+++ b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/config.rs@@ -10,9 +10,20 @@ pub struct Config {/// Number of users processed concurrently. Defaults to 1, capped at 50/// to bound DB connection usage.pub concurrency: usize,- /// When true, recomputes is_signal in both directions (clears stale true- /// flags) instead of the default set-true-only pass.- pub full_recompute: bool,+ /// What the per-link pass does; see [`BackfillMode`].+ pub mode: BackfillMode,+}++/// What the per-link pass does.+#[derive(Clone, Copy, PartialEq, Eq, Debug)]+pub enum BackfillMode {+ /// Flag matching threads true; never clears (default).+ SetTrueOnly,+ /// Clear + re-set per link so stale true flags are repaired.+ FullRecompute,+ /// Read-only: count threads whose is_signal disagrees with the+ /// heuristic. No writes.+ Verify,}env_vars! {@@ -20,6 +31,7 @@ env_vars! {struct DatabaseUrl;struct Concurrency;struct FullRecompute;+ struct Verify;}impl Config {@@ -38,6 +50,16 @@ impl Config {Some(v) => v.parse::<bool>().context("FULL_RECOMPUTE is not a bool")?,None => false,};+ let verify = match Verify::new().ok() {+ Some(v) => v.parse::<bool>().context("VERIFY is not a bool")?,+ None => false,+ };+ let mode = match (verify, full_recompute) {+ (true, true) => anyhow::bail!("VERIFY and FULL_RECOMPUTE are mutually exclusive"),+ (true, false) => BackfillMode::Verify,+ (false, true) => BackfillMode::FullRecompute,+ (false, false) => BackfillMode::SetTrueOnly,+ };Ok(Self {macro_ids: MacroIds::new().ok().map(|v| v.to_string()),@@ -45,7 +67,7 @@ impl Config {.context("DATABASE_URL not set")?.to_string(),concurrency,- full_recompute,+ mode,})}}diff --git a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/process.rs b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/process.rsindex bcb6d4a55e..67932e1e99 100644--- a/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/process.rs+++ b/rust/cloud-storage/email_service/src/bin/backfill_signal_flags/process.rs@@ -1,14 +1,16 @@+use crate::config::BackfillMode;use sqlx::types::Uuid;/// Flags every thread on one of the user's links that has at least one/// non-TRASH message matching the importance heuristic. One statement per/// link keeps transactions small and the progress output per-mailbox.-/// With `full_recompute`, stale true flags are also cleared (clear + re-set-/// in one transaction per link). Returns the number of threads flagged.+/// `FullRecompute` also clears stale true flags (clear + re-set in one+/// transaction per link); `Verify` is read-only and counts disagreements.+/// Returns the number of threads flagged (or mismatched, for `Verify`).pub async fn process_macro_id(pool: &sqlx::PgPool,macro_id: &str,- full_recompute: bool,+ mode: BackfillMode,) -> anyhow::Result<u64> {let link_ids: Vec<Uuid> =sqlx::query_scalar!("SELECT id FROM email_links WHERE macro_id = $1", macro_id)@@ -20,14 +22,21 @@ pub async fn process_macro_id(return Ok(0);}- let mut total_flagged = 0u64;+ let mut total = 0u64;for link_id in link_ids {+ if mode == BackfillMode::Verify {+ let mismatched = verify_link(pool, link_id).await?;+ println!("[{macro_id}] link {link_id}: {mismatched} threads mismatched");+ total += mismatched;+ continue;+ }+let mut tx = pool.begin().await?;// Full recompute: clear everything first so the set-true pass below// yields exactly the heuristic verdict. Same transaction, so readers// never observe the intermediate all-false state.- if full_recompute {+ if mode == BackfillMode::FullRecompute {let cleared = sqlx::query!(r#"UPDATE email_threads@@ -147,11 +156,122 @@ pub async fn process_macro_id(tx.commit().await?;// Prefix with the user so interleaved concurrent output stays readable.- println!("[{macro_id}] link {link_id}: flagged {flagged} threads");- total_flagged += flagged;+ // FullRecompute clears first, so its count is "threads that ended+ // true", not rows changed.+ if mode == BackfillMode::FullRecompute {+ println!("[{macro_id}] link {link_id}: computed {flagged} signal threads");+ } else {+ println!("[{macro_id}] link {link_id}: flagged {flagged} threads");+ }+ total += flagged;}- Ok(total_flagged)+ Ok(total)+}++/// Read-only differential check: counts the link's threads whose stored+/// is_signal disagrees with the live heuristic verdict. Mirrors+/// sync_thread_signal_flag (email_db_client/threads/update.rs).+async fn verify_link(pool: &sqlx::PgPool, link_id: Uuid) -> anyhow::Result<u64> {+ let mismatched = sqlx::query_scalar!(+ r#"+ SELECT count(*) AS "mismatched!"+ FROM email_threads t+ WHERE t.link_id = $1+ AND t.is_signal IS DISTINCT FROM EXISTS (+ SELECT 1+ FROM email_messages m+ WHERE m.thread_id = t.id+ AND NOT EXISTS (+ SELECT 1 FROM email_message_labels ml+ JOIN email_labels l ON ml.label_id = l.id+ WHERE ml.message_id = m.id AND l.name = 'TRASH'+ )+ AND (+ (+ EXISTS (+ SELECT 1+ FROM email_contacts sender_c+ JOIN email_filters ef+ ON ef.link_id = m.link_id+ AND ef.email_address IS NOT NULL+ AND LOWER(ef.email_address) = LOWER(sender_c.email_address)+ WHERE sender_c.id = m.from_contact_id+ AND ef.is_important = TRUE+ )+ OR EXISTS (+ SELECT 1+ FROM email_contacts sender_c+ JOIN email_filters ef+ ON ef.link_id = m.link_id+ AND ef.email_domain IS NOT NULL+ AND LOWER(ef.email_domain) = LOWER(SPLIT_PART(sender_c.email_address, '@', 2))+ WHERE sender_c.id = m.from_contact_id+ AND ef.is_important = TRUE+ AND NOT EXISTS (+ SELECT 1 FROM email_filters ef_addr+ WHERE ef_addr.link_id = m.link_id+ AND ef_addr.email_address IS NOT NULL+ AND LOWER(ef_addr.email_address) = LOWER(sender_c.email_address)+ AND ef_addr.is_important = FALSE+ )+ )+ )+ OR (+ NOT (+ EXISTS (+ SELECT 1+ FROM email_contacts sender_c+ JOIN email_filters ef+ ON ef.link_id = m.link_id+ AND ef.email_address IS NOT NULL+ AND LOWER(ef.email_address) = LOWER(sender_c.email_address)+ WHERE sender_c.id = m.from_contact_id+ AND ef.is_important = FALSE+ )+ OR EXISTS (+ SELECT 1+ FROM email_contacts sender_c+ JOIN email_filters ef+ ON ef.link_id = m.link_id+ AND ef.email_domain IS NOT NULL+ AND LOWER(ef.email_domain) = LOWER(SPLIT_PART(sender_c.email_address, '@', 2))+ WHERE sender_c.id = m.from_contact_id+ AND ef.is_important = FALSE+ AND NOT EXISTS (+ SELECT 1 FROM email_filters ef_addr+ WHERE ef_addr.link_id = m.link_id+ AND ef_addr.email_address IS NOT NULL+ AND LOWER(ef_addr.email_address) = LOWER(sender_c.email_address)+ AND ef_addr.is_important = TRUE+ )+ )+ )+ AND (+ m.is_draft = TRUE+ OR EXISTS (+ SELECT 1 FROM email_message_labels ml+ JOIN email_labels l ON ml.label_id = l.id+ WHERE ml.message_id = m.id+ AND l.name IN ('CATEGORY_PERSONAL', 'SENT', 'DRAFT')+ )+ OR NOT EXISTS (+ SELECT 1 FROM email_message_labels ml+ JOIN email_labels l ON ml.label_id = l.id+ WHERE ml.message_id = m.id+ AND l.name IN ('CATEGORY_UPDATES', 'CATEGORY_PROMOTIONS', 'CATEGORY_SOCIAL', 'CATEGORY_FORUMS')+ )+ )+ )+ )+ )+ "#,+ link_id+ )+ .fetch_one(pool)+ .await?;++ Ok(mismatched as u64)}/// Every macro ID that owns at least one email link. Connected secondarydiff --git a/rust/cloud-storage/item_filters/src/ast/email.rs b/rust/cloud-storage/item_filters/src/ast/email.rsindex 75b141749d..fdbdb9bf49 100644--- a/rust/cloud-storage/item_filters/src/ast/email.rs+++ b/rust/cloud-storage/item_filters/src/ast/email.rs@@ -53,7 +53,9 @@ pub enum EmailLiteral {Owner(Uuid),/// This value filters by project IDProjectId(String),- /// This node value filters by email importance. false short-circuits to match nothing.+ /// Filters by email importance, evaluated against the denormalized+ /// `email_threads.is_signal` flag: true matches signal threads, false+ /// matches noise threads.Importance(bool),/// This node value filters by notification done state for emails.NotificationDone(bool),