Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(search): backfill emails by thread id (#5472)
gbirmane4b3e565be54
diff --git a/.sqlx/query-dd51e47a73888eded59ccd5769444a157342ce5cfdb54f63d1982d1a14be1f73.json b/.sqlx/query-dd51e47a73888eded59ccd5769444a157342ce5cfdb54f63d1982d1a14be1f73.jsonnew file mode 100644index 0000000000..01348d91ec--- /dev/null+++ b/.sqlx/query-dd51e47a73888eded59ccd5769444a157342ce5cfdb54f63d1982d1a14be1f73.json@@ -0,0 +1,28 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT t.id, l.macro_id\n FROM email_threads t\n JOIN email_links l ON t.link_id = l.id\n WHERE t.id = ANY($1)\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "macro_id",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "UuidArray"+ ]+ },+ "nullable": [+ false,+ false+ ]+ },+ "hash": "dd51e47a73888eded59ccd5769444a157342ce5cfdb54f63d1982d1a14be1f73"+}diff --git a/crates/email_db_client/src/threads/get.rs b/crates/email_db_client/src/threads/get.rsindex d142c6c4f9..72146bcf74 100644--- a/crates/email_db_client/src/threads/get.rs+++ b/crates/email_db_client/src/threads/get.rs@@ -60,6 +60,33 @@ pub async fn get_paginated_thread_ids_with_macro_user_id_since(Ok(result)}+/// gets the macro user id for each of the given threads.+///+/// Unlike the paginated variants this neither sorts nor offsets: it is a+/// primary-key lookup, so it stays short enough to run on a read replica+/// without tripping a recovery conflict. Threads that do not exist are absent+/// from the result.+#[tracing::instrument(skip(pool, thread_ids), err)]+pub async fn get_thread_ids_with_macro_user_id_by_ids(+ pool: &PgPool,+ thread_ids: &[Uuid],+) -> anyhow::Result<Vec<(Uuid, String)>> {+ let result = sqlx::query!(+ r#"+ SELECT t.id, l.macro_id+ FROM email_threads t+ JOIN email_links l ON t.link_id = l.id+ WHERE t.id = ANY($1)+ "#,+ thread_ids+ )+ .map(|row| (row.id, row.macro_id))+ .fetch_all(pool)+ .await?;++ Ok(result)+}+/// get the ids of the latest-updated threads for the user.#[tracing::instrument(skip(pool), err)]pub async fn get_latest_thread_ids_paginated(diff --git a/services/search_processing_service/src/domain/models.rs b/services/search_processing_service/src/domain/models.rsindex 7dd0e8474a..256f92ec8a 100644--- a/services/search_processing_service/src/domain/models.rs+++ b/services/search_processing_service/src/domain/models.rs@@ -214,6 +214,10 @@ pub struct ProjectBackfillCursor {#[derive(Debug, Clone, Default, Deserialize)]#[serde(default)]pub struct EmailBackfillRequest {+ /// Reindex exactly these threads, ignoring `since`. Pages over the given+ /// ids rather than scanning the table, so a targeted repair costs one+ /// primary-key lookup per page.+ pub thread_ids: Vec<uuid::Uuid>,/// Only backfill threads updated at or after this instant.pub since: Option<DateTime<Utc>>,pub index_override: Option<String>,diff --git a/services/search_processing_service/src/outbound/source.rs b/services/search_processing_service/src/outbound/source.rsindex c2c77cab4b..24476efd96 100644--- a/services/search_processing_service/src/outbound/source.rs+++ b/services/search_processing_service/src/outbound/source.rs@@ -280,6 +280,29 @@ impl BackfillSource for PgBackfillSource {.filter(|n| *n > 0).unwrap_or(DEFAULT_EMAIL_BATCH_SIZE);+ // An explicit id list pages in memory: each page is a primary-key+ // lookup, so a targeted repair never runs the scan-and-sort that the+ // `since` and full variants depend on.+ if !req.thread_ids.is_empty() {+ let page = page_of(&req.thread_ids, offset, self.page_sizes.emails);+ if page.is_empty() {+ return Ok(SourcePage::empty());+ }+ let rows = email_db_client::threads::get::get_thread_ids_with_macro_user_id_by_ids(+ &self.db, page,+ )+ .await+ .map_err(BackfillError::Source)?;+ // Advance by the ids consumed, not the rows found, or unknown ids+ // would stall the drain loop on the same page forever.+ return Ok(email_source_page(+ rows,+ batch_size,+ page.len(),+ req.index_override.as_deref(),+ ));+ }+let rows = match req.since {Some(since) => {email_db_client::threads::get::get_paginated_thread_ids_with_macro_user_id_since(@@ -305,34 +328,12 @@ impl BackfillSource for PgBackfillSource {return Ok(SourcePage::empty());}- let mut by_user: HashMap<String, Vec<String>> = HashMap::new();- for (thread_id, macro_user_id) in rows {- by_user- .entry(macro_user_id)- .or_default()- .push(thread_id.to_string());- }-- let messages: Vec<SearchQueueMessage> = by_user- .into_iter()- .flat_map(|(macro_user_id, thread_ids)| {- thread_ids- .chunks(batch_size)- .map(|chunk| {- SearchQueueMessage::ExtractEmailThreadBatch(EmailThreadBatchMessage {- thread_ids: chunk.to_vec(),- macro_user_id: macro_user_id.clone(),- index_override: req.index_override.clone(),- })- })- .collect::<Vec<_>>()- })- .collect();-- Ok(SourcePage {- messages,+ Ok(email_source_page(+ rows,+ batch_size,rows_consumed,- })+ req.index_override.as_deref(),+ ))}async fn fetch_entity_properties(@@ -408,3 +409,52 @@ impl BackfillSource for PgBackfillSource {))}}++/// The `offset..offset + limit` window of `ids`, empty once the offset passes+/// the end so the drain loop terminates.+fn page_of(ids: &[uuid::Uuid], offset: usize, limit: usize) -> &[uuid::Uuid] {+ let start = offset.min(ids.len());+ let end = start.saturating_add(limit).min(ids.len());+ &ids[start..end]+}++/// Group resolved threads by owner and chunk each owner's threads into batch+/// messages. `rows_consumed` is what the drain loop advances its offset by.+fn email_source_page(+ rows: Vec<(uuid::Uuid, String)>,+ batch_size: usize,+ rows_consumed: usize,+ index_override: Option<&str>,+) -> SourcePage {+ let mut by_user: HashMap<String, Vec<String>> = HashMap::new();+ for (thread_id, macro_user_id) in rows {+ by_user+ .entry(macro_user_id)+ .or_default()+ .push(thread_id.to_string());+ }++ let messages: Vec<SearchQueueMessage> = by_user+ .into_iter()+ .flat_map(|(macro_user_id, thread_ids)| {+ thread_ids+ .chunks(batch_size)+ .map(|chunk| {+ SearchQueueMessage::ExtractEmailThreadBatch(EmailThreadBatchMessage {+ thread_ids: chunk.to_vec(),+ macro_user_id: macro_user_id.clone(),+ index_override: index_override.map(str::to_string),+ })+ })+ .collect::<Vec<_>>()+ })+ .collect();++ SourcePage {+ messages,+ rows_consumed,+ }+}++#[cfg(test)]+mod test;diff --git a/services/search_processing_service/src/outbound/source/test.rs b/services/search_processing_service/src/outbound/source/test.rsnew file mode 100644index 0000000000..4b8344ac1f--- /dev/null+++ b/services/search_processing_service/src/outbound/source/test.rs@@ -0,0 +1,98 @@+use super::*;++fn uuid(n: u128) -> uuid::Uuid {+ uuid::Uuid::from_u128(n)+}++fn thread_ids(page: &SourcePage) -> Vec<Vec<String>> {+ page.messages+ .iter()+ .map(|message| match message {+ SearchQueueMessage::ExtractEmailThreadBatch(batch) => batch.thread_ids.clone(),+ other => panic!("unexpected message: {other:?}"),+ })+ .collect()+}++#[test]+fn page_of_walks_the_id_list() {+ let ids: Vec<uuid::Uuid> = (0..5).map(uuid).collect();++ assert_eq!(page_of(&ids, 0, 2), &ids[0..2]);+ assert_eq!(page_of(&ids, 2, 2), &ids[2..4]);+ assert_eq!(page_of(&ids, 4, 2), &ids[4..5]);+}++#[test]+fn page_of_is_empty_past_the_end() {+ let ids: Vec<uuid::Uuid> = (0..3).map(uuid).collect();++ assert!(page_of(&ids, 3, 10).is_empty());+ assert!(page_of(&ids, 99, 10).is_empty());+}++#[test]+fn page_of_clamps_a_limit_beyond_the_end() {+ let ids: Vec<uuid::Uuid> = (0..3).map(uuid).collect();++ assert_eq!(page_of(&ids, 0, 100), &ids[..]);+}++#[test]+fn groups_threads_by_owner() {+ let page = email_source_page(+ vec![+ (uuid(1), "macro|[email protected]".into()),+ (uuid(2), "macro|[email protected]".into()),+ (uuid(3), "macro|[email protected]".into()),+ ],+ 50,+ 3,+ None,+ );++ assert_eq!(page.rows_consumed, 3);+ let mut sizes: Vec<usize> = thread_ids(&page).iter().map(Vec::len).collect();+ sizes.sort_unstable();+ assert_eq!(sizes, vec![1, 2]);+}++#[test]+fn chunks_each_owner_at_the_batch_size() {+ let rows: Vec<(uuid::Uuid, String)> = (0..120)+ .map(|n| (uuid(n), "macro|[email protected]".to_string()))+ .collect();++ let page = email_source_page(rows, 50, 120, None);++ let mut sizes: Vec<usize> = thread_ids(&page).iter().map(Vec::len).collect();+ sizes.sort_unstable();+ assert_eq!(sizes, vec![20, 50, 50]);+ assert_eq!(page.rows_consumed, 120);+}++#[test]+fn rows_consumed_is_independent_of_rows_found() {+ // Unknown ids resolve to nothing; the loop still has to advance past them.+ let page = email_source_page(vec![(uuid(1), "macro|[email protected]".into())], 50, 10, None);++ assert_eq!(page.rows_consumed, 10);+ assert_eq!(thread_ids(&page), vec![vec![uuid(1).to_string()]]);+}++#[test]+fn carries_the_index_override() {+ let page = email_source_page(+ vec![(uuid(1), "macro|[email protected]".into())],+ 50,+ 1,+ Some("emails_v2"),+ );++ match &page.messages[0] {+ SearchQueueMessage::ExtractEmailThreadBatch(batch) => {+ assert_eq!(batch.index_override.as_deref(), Some("emails_v2"));+ }+ other => panic!("unexpected message: {other:?}"),+ }+}