Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(email): backfill Outlook reply snippets from stored bodies (#8801)
teo64b3224555a6
diff --git a/.sqlx/query-a89cd1b96443a2730aa2d74bacdc04f08cd73db88ddc7f27ea2e51c3cbef1f4c.json b/.sqlx/query-a89cd1b96443a2730aa2d74bacdc04f08cd73db88ddc7f27ea2e51c3cbef1f4c.jsonnew file mode 100644index 0000000000..006c7af109--- /dev/null+++ b/.sqlx/query-a89cd1b96443a2730aa2d74bacdc04f08cd73db88ddc7f27ea2e51c3cbef1f4c.json@@ -0,0 +1,54 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT id, provider_id AS \"provider_id!\", subject, body_html_sanitized, body_text, snippet\n FROM email_messages\n WHERE link_id = $1 AND provider_id > $2\n ORDER BY provider_id\n LIMIT $3\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "provider_id!",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "subject",+ "type_info": "Text"+ },+ {+ "ordinal": 3,+ "name": "body_html_sanitized",+ "type_info": "Text"+ },+ {+ "ordinal": 4,+ "name": "body_text",+ "type_info": "Text"+ },+ {+ "ordinal": 5,+ "name": "snippet",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Text",+ "Int8"+ ]+ },+ "nullable": [+ false,+ true,+ true,+ true,+ true,+ true+ ]+ },+ "hash": "a89cd1b96443a2730aa2d74bacdc04f08cd73db88ddc7f27ea2e51c3cbef1f4c"+}diff --git a/.sqlx/query-c0d8950d7b3e655a1e255c5a401a95979ecb4135f2cfc302a67d37ab7a3cf88c.json b/.sqlx/query-c0d8950d7b3e655a1e255c5a401a95979ecb4135f2cfc302a67d37ab7a3cf88c.jsonnew file mode 100644index 0000000000..a61972b508--- /dev/null+++ b/.sqlx/query-c0d8950d7b3e655a1e255c5a401a95979ecb4135f2cfc302a67d37ab7a3cf88c.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE email_messages m SET snippet = v.snippet\n FROM UNNEST($1::uuid[], $2::text[], $3::text[]) AS v(id, stored, snippet)\n WHERE m.id = v.id AND m.snippet IS NOT DISTINCT FROM v.stored\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "UuidArray",+ "TextArray",+ "TextArray"+ ]+ },+ "nullable": []+ },+ "hash": "c0d8950d7b3e655a1e255c5a401a95979ecb4135f2cfc302a67d37ab7a3cf88c"+}diff --git a/.sqlx/query-e2611e3ade00553fd9cb2dcac5681c43e1c26d39d98addbb636392024a7c5dec.json b/.sqlx/query-e2611e3ade00553fd9cb2dcac5681c43e1c26d39d98addbb636392024a7c5dec.jsonnew file mode 100644index 0000000000..2017a4df21--- /dev/null+++ b/.sqlx/query-e2611e3ade00553fd9cb2dcac5681c43e1c26d39d98addbb636392024a7c5dec.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT id FROM email_links WHERE provider = 'OUTLOOK' AND ($1::uuid[] IS NULL OR id = ANY($1)) ORDER BY id",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "UuidArray"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "e2611e3ade00553fd9cb2dcac5681c43e1c26d39d98addbb636392024a7c5dec"+}diff --git a/crates/email_api_client/src/outbound/outlook/messages.rs b/crates/email_api_client/src/outbound/outlook/messages.rsindex 168ed6ad50..774801f2d3 100644--- a/crates/email_api_client/src/outbound/outlook/messages.rs+++ b/crates/email_api_client/src/outbound/outlook/messages.rs@@ -305,25 +305,6 @@ fn contact(recipient: wire::Recipient) -> ContactInfo {}}-/// Graph's `bodyPreview` length.-const SNIPPET_CHARS: usize = 255;--fn reply_snippet(subject: Option<&str>, html: Option<&str>, text: Option<&str>) -> Option<String> {- let reply = email_utils::body_replyless::compute_body_replyless(subject, html, text)?;- let reply = match html {- Some(_) => email_utils::body_parsed::html_to_plaintext(&reply)?,- None => reply,- };- let snippet: String = reply- .split_whitespace()- .collect::<Vec<_>>()- .join(" ")- .chars()- .take(SNIPPET_CHARS)- .collect();- (!snippet.is_empty()).then_some(snippet)-}-pub(super) fn normalize(source: wire::Message,attachments: Vec<Attachment>,@@ -358,7 +339,7 @@ pub(super) fn normalize(Some(body) => (Some(body.content), None),None => (None, None),};- let snippet = reply_snippet(+ let snippet = email_utils::body_replyless::reply_snippet(source.subject.as_deref(),body_html_sanitized.as_deref(),body_text.as_deref(),diff --git a/crates/email_utils/AGENTS.md b/crates/email_utils/AGENTS.mdindex 77ff612c8d..9f9b96e06e 100644--- a/crates/email_utils/AGENTS.md+++ b/crates/email_utils/AGENTS.md@@ -12,7 +12,8 @@ sender/generic-address checks.`sanitize_html_fragment` — every HTML body that reaches`body_html_sanitized` must pass through one of these.- `src/body_replyless.rs` + `src/body_replyless/`: strips quoted reply chains- from HTML and plaintext bodies.+ from HTML and plaintext bodies; `reply_snippet` is the Outlook snippet that+ ingest and `backfill_outlook_snippets` both derive from them.- `src/body_parsed.rs`: the parsed-body representation used across callers.## Scopediff --git a/crates/email_utils/src/body_replyless.rs b/crates/email_utils/src/body_replyless.rsindex 7d7b40a114..ce13f67017 100644--- a/crates/email_utils/src/body_replyless.rs+++ b/crates/email_utils/src/body_replyless.rs@@ -7,6 +7,9 @@ mod test;use html_parser::extract_reply_html;use plaintext_parser::extract_reply_plaintext;+/// Graph's `bodyPreview` length.+const SNIPPET_CHARS: usize = 255;+/// Computes the body of a message with reply/forwarded-thread content stripped.////// Returns `None` if the message has neither HTML nor plaintext body content.@@ -21,3 +24,23 @@ pub fn compute_body_replyless(body_text.map(|body| extract_reply_plaintext(subject, body))}}++pub fn reply_snippet(+ subject: Option<&str>,+ body_html_sanitized: Option<&str>,+ body_text: Option<&str>,+) -> Option<String> {+ let reply = compute_body_replyless(subject, body_html_sanitized, body_text)?;+ let reply = match body_html_sanitized {+ Some(_) => crate::body_parsed::html_to_plaintext(&reply)?,+ None => reply,+ };+ let snippet: String = reply+ .split_whitespace()+ .collect::<Vec<_>>()+ .join(" ")+ .chars()+ .take(SNIPPET_CHARS)+ .collect();+ (!snippet.is_empty()).then_some(snippet)+}diff --git a/services/email_service/Cargo.toml b/services/email_service/Cargo.tomlindex edbc2cb25d..d51ea247a1 100644--- a/services/email_service/Cargo.toml+++ b/services/email_service/Cargo.toml@@ -21,6 +21,10 @@ path = "src/bin/backfill_calendar_flags/backfill_calendar_flags.rs"name = "backfill_signal_flags"path = "src/bin/backfill_signal_flags/backfill_signal_flags.rs"+[[bin]]+name = "backfill_outlook_snippets"+path = "src/bin/backfill_outlook_snippets/backfill_outlook_snippets.rs"+[[bin]]name = "backfill_gmail_attachments"path = "src/bin/backfill_gmail_attachments/backfill_gmail_attachments.rs"diff --git a/services/email_service/src/bin/backfill_outlook_snippets/backfill_outlook_snippets.rs b/services/email_service/src/bin/backfill_outlook_snippets/backfill_outlook_snippets.rsnew file mode 100644index 0000000000..512c7c172f--- /dev/null+++ b/services/email_service/src/bin/backfill_outlook_snippets/backfill_outlook_snippets.rs@@ -0,0 +1,72 @@+//! # Outlook Reply Snippet Backfill+//!+//! Recomputes `email_messages.snippet` for Outlook messages from their stored+//! bodies, using the reply extraction Outlook ingest now uses. Messages+//! ingested before that change kept Graph's `bodyPreview`, which starts with+//! the quoted message being replied to, and ingest skips them while their+//! ETag is unchanged. Messages with no body keep their snippet, as ingest+//! does. A rerun changes nothing.+//!+//! ## Required Environment Variables:+//! - `DATABASE_URL`: The connection string for the PostgreSQL database.+//!+//! ## Optional Environment Variables:+//! - `LINK_IDS`: Comma-separated link IDs to backfill. Only Outlook links+//! among them are processed. When unset, every Outlook link is processed.+//! - `CONCURRENCY`: Number of links processed concurrently (defaults to 1,+//! capped at 50).+//! - `VERIFY`: When `true`, read-only mode — reports messages whose snippet+//! would change and exits non-zero if any are found.++mod config;+mod process;++use macro_entrypoint::MacroEntrypoint;+use rootcause::prelude::{Report, ResultExt as _};+use sqlx::postgres::PgPoolOptions;++#[tokio::main]+async fn main() -> Result<(), Report> {+ let entrypoint = MacroEntrypoint::default().init();+ let result = run().await;+ entrypoint.shutdown();+ result+}++async fn run() -> Result<(), Report> {+ let config = config::Config::from_env()?;+ let db_pool = PgPoolOptions::new()+ .min_connections(1)+ .max_connections(config.concurrency as u32)+ .connect(&config.database_url)+ .await+ .context("could not connect to the database")?;++ let mode = config.mode;+ let results = process::backfill(&db_pool, config.link_ids, config.concurrency, mode).await?;++ let total: u64 = results.iter().filter_map(|(_, r)| r.as_ref().ok()).sum();+ let failures: Vec<String> = results+ .iter()+ .filter(|(_, r)| r.is_err())+ .map(|(id, _)| id.to_string())+ .collect();++ tracing::info!(+ ?mode,+ snippets = total,+ failures = failures.len(),+ "all links processed"+ );+ if !failures.is_empty() {+ rootcause::bail!(+ "{} links failed ({}); rerun with LINK_IDS to retry them",+ failures.len(),+ failures.join(", ")+ );+ }+ if mode == config::Mode::Verify && total > 0 {+ rootcause::bail!("{total} Outlook snippets are stale; run without VERIFY to repair");+ }+ Ok(())+}diff --git a/services/email_service/src/bin/backfill_outlook_snippets/config.rs b/services/email_service/src/bin/backfill_outlook_snippets/config.rsnew file mode 100644index 0000000000..7b8d92d929--- /dev/null+++ b/services/email_service/src/bin/backfill_outlook_snippets/config.rs@@ -0,0 +1,66 @@+use macro_env_var::{env_vars, maybe_env_vars};+use rootcause::prelude::{Report, ResultExt as _};+use uuid::Uuid;++/// Holds all configuration loaded from environment variables.+pub struct Config {+ pub database_url: DatabaseUrl,+ /// Links to backfill. When unset, every Outlook link is processed.+ pub link_ids: Option<Vec<Uuid>>,+ /// Number of links processed concurrently. Defaults to 1, capped at 50+ /// to bound DB connection usage.+ pub concurrency: usize,+ pub mode: Mode,+}++/// What the per-link pass does.+#[derive(Clone, Copy, PartialEq, Eq, Debug)]+pub enum Mode {+ /// Rewrite stale snippets (default).+ Apply,+ /// Read-only: count stale snippets. No writes.+ Verify,+}++env_vars! {+ pub struct DatabaseUrl;+}++maybe_env_vars! {+ struct LinkIds;+ struct Concurrency;+ struct Verify;+}++impl Config {+ /// Creates a new `Config` instance by reading from environment variables.+ /// Returns an error if any required variable is not set.+ pub fn from_env() -> Result<Self, Report> {+ let link_ids = LinkIds::new()+ .map(|ids| {+ ids.split(',')+ .map(|id| Uuid::parse_str(id.trim()))+ .collect::<Result<Vec<_>, _>>()+ .context("LINK_IDS is not a comma-separated list of UUIDs")+ })+ .transpose()?;+ let concurrency = Concurrency::new()+ .map(|n| n.parse::<usize>().context("CONCURRENCY is not a number"))+ .transpose()?+ .map_or(1, |n| n.clamp(1, 50));+ let verify = Verify::new()+ .map(|v| v.parse::<bool>().context("VERIFY is not a bool"))+ .transpose()?;++ Ok(Self {+ database_url: DatabaseUrl::new()?,+ link_ids,+ concurrency,+ mode: if verify == Some(true) {+ Mode::Verify+ } else {+ Mode::Apply+ },+ })+ }+}diff --git a/services/email_service/src/bin/backfill_outlook_snippets/process.rs b/services/email_service/src/bin/backfill_outlook_snippets/process.rsnew file mode 100644index 0000000000..f7b4aedd20--- /dev/null+++ b/services/email_service/src/bin/backfill_outlook_snippets/process.rs@@ -0,0 +1,120 @@+use crate::config::Mode;+use email_utils::body_replyless::reply_snippet;+use futures::stream::{self, StreamExt};+use rootcause::Report;+use sqlx::PgPool;+use uuid::Uuid;++#[cfg(test)]+mod test;++const PAGE_SIZE: i64 = 500;++/// A link and how many of its snippets were stale, or why it failed.+pub type LinkOutcome = (Uuid, Result<u64, Report>);++/// Backfills each Outlook link among `link_ids` (every Outlook link when+/// `None`) and returns each link's outcome.+#[tracing::instrument(+ name = "email.backfill_outlook_snippets",+ skip(pool, link_ids),+ fields(links),+ err+)]+pub async fn backfill(+ pool: &PgPool,+ link_ids: Option<Vec<Uuid>>,+ concurrency: usize,+ mode: Mode,+) -> Result<Vec<LinkOutcome>, Report> {+ let link_ids = sqlx::query_scalar!(+ "SELECT id FROM email_links WHERE provider = 'OUTLOOK' AND ($1::uuid[] IS NULL OR id = ANY($1)) ORDER BY id",+ link_ids.as_deref()+ )+ .fetch_all(pool)+ .await?;+ tracing::Span::current().record("links", link_ids.len());++ Ok(stream::iter(link_ids)+ .map(|link_id| async move { (link_id, backfill_link(pool, link_id, mode).await) })+ .buffer_unordered(concurrency)+ .collect()+ .await)+}++/// Pages the link's provider messages by `provider_id`, the only order the+/// `(link_id, provider_id)` index serves, and rewrites each snippet that+/// differs from the one ingest now derives from the same stored bodies. Rows+/// Outlook never sent (no `provider_id`) are left alone.+#[tracing::instrument(+ name = "email.backfill_outlook_snippets.link",+ skip(pool),+ fields(pages = 0u64, snippets),+ err+)]+async fn backfill_link(pool: &PgPool, link_id: Uuid, mode: Mode) -> Result<u64, Report> {+ let mut after = String::new();+ let mut pages = 0u64;+ let mut stale = 0;+ loop {+ let page = sqlx::query!(+ r#"+ SELECT id, provider_id AS "provider_id!", subject, body_html_sanitized, body_text, snippet+ FROM email_messages+ WHERE link_id = $1 AND provider_id > $2+ ORDER BY provider_id+ LIMIT $3+ "#,+ link_id,+ after,+ PAGE_SIZE+ )+ .fetch_all(pool)+ .await?;+ let Some(last) = page.last() else {+ let span = tracing::Span::current();+ span.record("pages", pages);+ span.record("snippets", stale);+ tracing::info!(snippets = stale, "link done");+ return Ok(stale);+ };+ after = last.provider_id.clone();+ pages += 1;++ let mut ids = Vec::new();+ let mut stored = Vec::new();+ let mut snippets = Vec::new();+ for row in page {+ let Some(snippet) = reply_snippet(+ row.subject.as_deref(),+ row.body_html_sanitized.as_deref(),+ row.body_text.as_deref(),+ ) else {+ continue;+ };+ if row.snippet.as_ref() != Some(&snippet) {+ ids.push(row.id);+ stored.push(row.snippet);+ snippets.push(snippet);+ }+ }++ stale += match mode {+ Mode::Verify => ids.len() as u64,+ // The stored-snippet guard skips rows ingest rewrote since the read.+ Mode::Apply => sqlx::query!(+ r#"+ UPDATE email_messages m SET snippet = v.snippet+ FROM UNNEST($1::uuid[], $2::text[], $3::text[]) AS v(id, stored, snippet)+ WHERE m.id = v.id AND m.snippet IS NOT DISTINCT FROM v.stored+ "#,+ &ids,+ &stored as &[Option<String>],+ &snippets+ )+ .execute(pool)+ .await?+ .rows_affected(),+ };+ }+}diff --git a/services/email_service/src/bin/backfill_outlook_snippets/process/test.rs b/services/email_service/src/bin/backfill_outlook_snippets/process/test.rsnew file mode 100644index 0000000000..97ce97f4a2--- /dev/null+++ b/services/email_service/src/bin/backfill_outlook_snippets/process/test.rs@@ -0,0 +1,169 @@+use super::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;++const OUTLOOK_LINK: Uuid = uuid::uuid!("0190a000-0000-7000-8000-000000000001");+const GMAIL_LINK: Uuid = uuid::uuid!("0190a000-0000-7000-8000-000000000002");++const QUOTED_PREVIEW: &str =+ "Tell sam I said hello From: Riley Sent: Tuesday To: Sam Subject: Hello Original message";++async fn seed(db: &PgPool) {+ let thread = uuid::uuid!("0190a000-0000-7000-8000-0000000000a1");+ let gmail_thread = uuid::uuid!("0190a000-0000-7000-8000-0000000000a2");+ sqlx::query!(+ "INSERT INTO email_links(id,macro_id,fusionauth_user_id,email_address,provider) VALUES ($1,'macro|[email protected]','outlook','[email protected]','OUTLOOK'),($2,'macro|[email protected]','gmail','[email protected]','GMAIL')",+ OUTLOOK_LINK,+ GMAIL_LINK+ )+ .execute(db)+ .await+ .unwrap();+ sqlx::query!(+ "INSERT INTO email_threads(id,link_id) VALUES ($1,$2),($3,$4)",+ thread,+ OUTLOOK_LINK,+ gmail_thread,+ GMAIL_LINK+ )+ .execute(db)+ .await+ .unwrap();+ let reply_html = email_utils::sanitize_email_html(concat!(+ "<html><body><div>Tell sam I said\nhello</div>",+ "<div id=\"appendonsend\"></div><hr><div id=\"divRplyFwdMsg\">",+ "<b>From:</b> Riley<br><b>Sent:</b> Tuesday<br><b>To:</b> Sam<br>",+ "<b>Subject:</b> Hello</div><div>Original message</div></body></html>"+ ));+ let reply_text = "Tell sam I said hello\n**From:** Riley\n**Sent:** Tuesday\n**To:** Sam\n**Subject:** Hello\nOriginal message";+ for (link, thread, provider_id, is_draft, html, text, snippet) in [+ (+ OUTLOOK_LINK,+ thread,+ Some("outlook-html-reply"),+ false,+ Some(reply_html.as_str()),+ Some(reply_text),+ Some(QUOTED_PREVIEW),+ ),+ (+ OUTLOOK_LINK,+ thread,+ Some("outlook-text-reply"),+ false,+ None,+ Some(+ "Me too\n\nOn Tue, Oct 20, 2026 at 3:00 PM Riley <[email protected]> wrote:\n> God I love outlook",+ ),+ Some(+ "Me too On Tue, Oct 20, 2026 at 3:00 PM Riley <[email protected]> wrote: > God I love outlook",+ ),+ ),+ (+ OUTLOOK_LINK,+ thread,+ Some("outlook-no-body"),+ false,+ None,+ None,+ Some("Graph preview kept"),+ ),+ (+ OUTLOOK_LINK,+ thread,+ None,+ true,+ Some("<div>Draft Outlook has not seen</div>"),+ Some("Draft Outlook has not seen"),+ None,+ ),+ (+ GMAIL_LINK,+ gmail_thread,+ Some("gmail-reply"),+ false,+ Some(reply_html.as_str()),+ Some(reply_text),+ Some(QUOTED_PREVIEW),+ ),+ ] {+ sqlx::query!(+ "INSERT INTO email_messages(id,link_id,thread_id,provider_id,is_draft,subject,body_html_sanitized,body_text,snippet) VALUES($1,$2,$3,$4,$5,'RE: Hello',$6,$7,$8)",+ Uuid::now_v7(),+ link,+ thread,+ provider_id,+ is_draft,+ html,+ text,+ snippet+ )+ .execute(db)+ .await+ .unwrap();+ }+}++async fn snippets(db: &PgPool) -> Vec<(Option<String>, Option<String>)> {+ sqlx::query!("SELECT provider_id, snippet FROM email_messages ORDER BY provider_id NULLS FIRST")+ .fetch_all(db)+ .await+ .unwrap()+ .into_iter()+ .map(|row| (row.provider_id, row.snippet))+ .collect()+}++async fn run(db: &PgPool, mode: Mode) -> Vec<(Uuid, u64)> {+ backfill(db, None, 1, mode)+ .await+ .unwrap()+ .into_iter()+ .map(|(link, result)| (link, result.unwrap()))+ .collect()+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn rewrites_only_outlook_replies_that_have_a_body(db: PgPool) {+ seed(&db).await;++ assert_eq!(run(&db, Mode::Apply).await, vec![(OUTLOOK_LINK, 2)]);++ assert_eq!(+ snippets(&db).await,+ vec![+ (None, None),+ (Some("gmail-reply".into()), Some(QUOTED_PREVIEW.into())),+ (+ Some("outlook-html-reply".into()),+ Some("Tell sam I said hello".into())+ ),+ (+ Some("outlook-no-body".into()),+ Some("Graph preview kept".into())+ ),+ (Some("outlook-text-reply".into()), Some("Me too".into())),+ ]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn verify_counts_stale_snippets_without_writing(db: PgPool) {+ seed(&db).await;+ let before = snippets(&db).await;++ assert_eq!(run(&db, Mode::Verify).await, vec![(OUTLOOK_LINK, 2)]);++ assert_eq!(snippets(&db).await, before);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_second_run_changes_nothing(db: PgPool) {+ seed(&db).await;+ assert_eq!(run(&db, Mode::Apply).await, vec![(OUTLOOK_LINK, 2)]);+ let after_first = snippets(&db).await;++ assert_eq!(run(&db, Mode::Apply).await, vec![(OUTLOOK_LINK, 0)]);+ assert_eq!(run(&db, Mode::Verify).await, vec![(OUTLOOK_LINK, 0)]);++ assert_eq!(snippets(&db).await, after_first);+}