Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(email): cut the cost of finishing a mailbox backfill (#9039)
teodd314cafebbf
diff --git a/crates/contacts/src/domain/service.rs b/crates/contacts/src/domain/service.rsindex 980677b282..f04f5906ad 100644--- a/crates/contacts/src/domain/service.rs+++ b/crates/contacts/src/domain/service.rs@@ -91,6 +91,10 @@ impl<R: ContactsRepository, N: ContactsNotifier> ContactsService for ContactsDom}}+/// How many relationships go in one contacts message. SQS rejects message+/// bodies over 256 KB, which a mailbox's whole contact list can exceed.+const CONNECTIONS_PER_MESSAGE: usize = 500;+/// Queue-backed implementation of [`ContactsIngress`].////// Serialises complete-graph or explicit-relationship messages and publishes@@ -113,9 +117,14 @@ impl<Q: ContactsIngressQueue> ContactsIngress for SqsContactsIngress<Q> {&self,connections: Vec<ContactConnection>,) -> Result<(), Report> {- self.queue- .publish_connections(ContactConnections::new(connections))- .await+ // Each message is upserted on its own, so a retry after a partial+ // publish only repeats upserts.+ for chunk in connections.chunks(CONNECTIONS_PER_MESSAGE) {+ self.queue+ .publish_connections(ContactConnections::new(chunk.to_vec()))+ .await?;+ }+ Ok(())}}diff --git a/crates/email_db_client/src/attachments/provider/upload.rs b/crates/email_db_client/src/attachments/provider/upload.rsindex 225a187691..60bc8e58d0 100644--- a/crates/email_db_client/src/attachments/provider/upload.rs+++ b/crates/email_db_client/src/attachments/provider/upload.rs@@ -147,9 +147,14 @@ pub async fn thread_media_atts_for_backfill(/// of job completion, as we don't know everyone the user has sent messages to until all their/// messages have been backfilled.///-/// There will be overlap between the attachments returned by this query and the ones returned by-/// fetch_attachment_threads_for_backfill, but the BackfillAttachment job has logic to ensure-/// duplicate attachments are not inserted by checking the DocumentEmail table.+/// Attachments that already have a live document are left out, using the same check the+/// BackfillAttachment job runs before importing. Imports from+/// fetch_attachment_threads_for_backfill that are still in flight can still overlap; the job's+/// check skips those.+///+/// This reads the whole mailbox once per backfill, so it starts from the mailbox's document+/// attachments and only reads the threads that hold them, instead of every message in every+/// thread with an attachment.#[tracing::instrument(skip(db), err)]#[allow(clippy::disallowed_methods, reason = "legacy code. fix later")]pub async fn fetch_job_attachments_for_backfill(@@ -159,74 +164,99 @@ pub async fn fetch_job_attachments_for_backfill(let query = format!(r#"WITH- eligible_threads AS (- SELECT DISTINCT thread_id- FROM public.email_messages- WHERE link_id = $1- AND has_attachments = true- ),- self_contact AS (- SELECT c.id- FROM public.email_links l- JOIN public.email_contacts c- ON c.link_id = l.id- AND LOWER(c.email_address) = LOWER(l.email_address)- WHERE l.id = $1+ -- Attachment rows only exist on messages flagged has_attachments, so+ -- the partial index on that flag finds every document. Carrying the+ -- output columns here avoids reading each message row a second time.+ mailbox_documents AS MATERIALIZED (+ SELECT+ a.id AS attachment_db_id,+ m.provider_id AS email_provider_id,+ a.provider_attachment_id,+ a.filename,+ a.mime_type,+ m.internal_date_ts,+ m.id AS message_db_id,+ m.thread_id AS thread_db_id,+ m.from_contact_id,+ m.subject+ FROM public.email_messages m+ JOIN public.email_attachments a ON a.message_id = m.id+ WHERE m.link_id = $1+ AND m.has_attachments = true+ AND a.filename IS NOT NULL+ AND NOT EXISTS (+ SELECT 1+ FROM public.document_email de+ JOIN public."Document" d ON d.id = de.document_id+ WHERE de.email_attachment_id = a.id+ AND d."deletedAt" IS NULL+ )+ -- attachment mime type filters injected below+ {}),- contacted AS (- SELECT DISTINCT emr.contact_id+ -- Everyone the user has sent mail to, except the user.+ contacted AS MATERIALIZED (+ SELECT DISTINCT recipient.contact_idFROM public.email_messages sent_message- JOIN public.email_message_recipients emr ON emr.message_id = sent_message.id+ JOIN public.email_message_recipients recipient ON recipient.message_id = sent_message.idWHERE sent_message.link_id = $1AND sent_message.is_sent = true+ AND NOT EXISTS (+ SELECT 1+ FROM public.email_links l+ JOIN public.email_contacts c+ ON c.link_id = l.id+ AND LOWER(c.email_address) = LOWER(l.email_address)+ WHERE l.id = $1+ AND c.id = recipient.contact_id+ )),- participants AS (- SELECT message.thread_id, message.from_contact_id AS contact_id- FROM eligible_threads eligible- JOIN public.email_messages message ON message.thread_id = eligible.thread_id- WHERE message.from_contact_id IS NOT NULL-- -- UNION ALL, not UNION: `qualified_threads` already applies- -- SELECT DISTINCT to this result, so a set-union dedupe here- -- sorts the whole participant set only to have the work thrown- -- away. Duplicates cannot change the final thread set.- UNION ALL-- SELECT message.thread_id, recipient.contact_id- FROM eligible_threads eligible- JOIN public.email_messages message ON message.thread_id = eligible.thread_id- JOIN public.email_message_recipients recipient ON recipient.message_id = message.id+ document_threads AS MATERIALIZED (+ SELECT DISTINCT thread_db_id AS thread_id+ FROM mailbox_documents+ ),+ -- Senders come from an index alone; most threads qualify here.+ sender_qualified_threads AS MATERIALIZED (+ SELECT DISTINCT sender_message.thread_id+ FROM document_threads+ JOIN public.email_messages sender_message ON sender_message.thread_id = document_threads.thread_id+ JOIN contacted ON contacted.contact_id = sender_message.from_contact_id),+ -- Recipients need each message row, so only check the threads no+ -- sender qualified.qualified_threads AS (- SELECT DISTINCT participant.thread_id- FROM participants participant- JOIN contacted ON contacted.contact_id = participant.contact_id+ SELECT thread_id+ FROM sender_qualified_threads++ UNION++ SELECT thread_message.thread_id+ FROM document_threads+ JOIN public.email_messages thread_message ON thread_message.thread_id = document_threads.thread_id+ JOIN public.email_message_recipients recipient ON recipient.message_id = thread_message.id+ JOIN contacted ON contacted.contact_id = recipient.contact_idWHERE NOT EXISTS (SELECT 1- FROM self_contact- WHERE self_contact.id = participant.contact_id+ FROM sender_qualified_threads+ WHERE sender_qualified_threads.thread_id = document_threads.thread_id))SELECT- a.id AS attachment_db_id,- m.provider_id as email_provider_id,- a.provider_attachment_id as provider_attachment_id,- a.filename as filename,- a.mime_type as mime_type,- m.internal_date_ts as internal_date_ts,- m.id as message_db_id,- m.thread_id as thread_db_id,- from_contact.email_address as sender_email,- m.subject as subject- FROM public.email_attachments a- JOIN public.email_messages m ON a.message_id = m.id- JOIN public.email_contacts from_contact ON m.from_contact_id = from_contact.id- WHERE m.thread_id IN (SELECT thread_id FROM qualified_threads)- -- attachment mime type filters injected below- {}- AND a.filename IS NOT NULL- ORDER BY m.internal_date_ts DESC+ document.attachment_db_id,+ document.email_provider_id,+ document.provider_attachment_id,+ document.filename,+ document.mime_type,+ document.internal_date_ts,+ document.message_db_id,+ document.thread_db_id,+ from_contact.email_address AS sender_email,+ document.subject+ FROM mailbox_documents document+ JOIN qualified_threads ON qualified_threads.thread_id = document.thread_db_id+ JOIN public.email_contacts from_contact ON from_contact.id = document.from_contact_id+ ORDER BY document.internal_date_ts DESC"#,ATTACHMENT_MIME_TYPE_FILTERS);diff --git a/services/email_service/src/util/sync_contacts.rs b/services/email_service/src/util/sync_contacts.rsindex e1437d9b7e..72d0bf9fef 100644--- a/services/email_service/src/util/sync_contacts.rs+++ b/services/email_service/src/util/sync_contacts.rs@@ -67,9 +67,6 @@ pub async fn sync_contacts<B: MacroEventBroker, I: ContactsIngress>(Ok(())}-/// How many address-book connections go in one contacts message.-const CONNECTIONS_PER_MESSAGE: usize = 500;-/// The connections from the link's owner to each distinct, non-automated/// address in `contacts`.fn address_book_connections(link: &Link, contacts: &[Contact]) -> Vec<ContactConnection> {@@ -107,18 +104,15 @@ async fn connect_address_book<I: ContactsIngress>(if cfg!(not(feature = "contacts_sync")) {return Ok(());}- for chunk in address_book_connections(link, contacts).chunks(CONNECTIONS_PER_MESSAGE) {- contacts_ingress- .enqueue_contact_connections(chunk.to_vec())- .await- .map_err(|error| {- anyhow!("{error:?}").context(format!(- "Unable to connect address book contacts for link_id: {}",- link.id- ))- })?;- }- Ok(())+ contacts_ingress+ .enqueue_contact_connections(address_book_connections(link, contacts))+ .await+ .map_err(|error| {+ anyhow!("{error:?}").context(format!(+ "Unable to connect address book contacts for link_id: {}",+ link.id+ ))+ })}/// Retrieves the current contact and other_contact sync tokens from the database for a given link.