Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(search): backfill emails by thread id (#5472)

gbirmane4b3e565be54
diff --git a/.sqlx/query-dd51e47a73888eded59ccd5769444a157342ce5cfdb54f63d1982d1a14be1f73.json b/.sqlx/query-dd51e47a73888eded59ccd5769444a157342ce5cfdb54f63d1982d1a14be1f73.json
new file mode 100644
index 0000000000..01348d91ec
--- /dev/null
+++ b/.sqlx/query-dd51e47a73888eded59ccd5769444a157342ce5cfdb54f63d1982d1a14be1f73.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT t.id, l.macro_id\n FROM email_threads t\n JOIN email_links l ON t.link_id = l.id\n WHERE t.id = ANY($1)\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "macro_id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "UuidArray"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "dd51e47a73888eded59ccd5769444a157342ce5cfdb54f63d1982d1a14be1f73"
+}
diff --git a/crates/email_db_client/src/threads/get.rs b/crates/email_db_client/src/threads/get.rs
index d142c6c4f9..72146bcf74 100644
--- a/crates/email_db_client/src/threads/get.rs
+++ b/crates/email_db_client/src/threads/get.rs
@@ -60,6 +60,33 @@ pub async fn get_paginated_thread_ids_with_macro_user_id_since(
Ok(result)
}
+/// gets the macro user id for each of the given threads.
+///
+/// Unlike the paginated variants this neither sorts nor offsets: it is a
+/// primary-key lookup, so it stays short enough to run on a read replica
+/// without tripping a recovery conflict. Threads that do not exist are absent
+/// from the result.
+#[tracing::instrument(skip(pool, thread_ids), err)]
+pub async fn get_thread_ids_with_macro_user_id_by_ids(
+ pool: &PgPool,
+ thread_ids: &[Uuid],
+) -> anyhow::Result<Vec<(Uuid, String)>> {
+ let result = sqlx::query!(
+ r#"
+ SELECT t.id, l.macro_id
+ FROM email_threads t
+ JOIN email_links l ON t.link_id = l.id
+ WHERE t.id = ANY($1)
+ "#,
+ thread_ids
+ )
+ .map(|row| (row.id, row.macro_id))
+ .fetch_all(pool)
+ .await?;
+
+ Ok(result)
+}
+
/// get the ids of the latest-updated threads for the user.
#[tracing::instrument(skip(pool), err)]
pub async fn get_latest_thread_ids_paginated(
diff --git a/services/search_processing_service/src/domain/models.rs b/services/search_processing_service/src/domain/models.rs
index 7dd0e8474a..256f92ec8a 100644
--- a/services/search_processing_service/src/domain/models.rs
+++ b/services/search_processing_service/src/domain/models.rs
@@ -214,6 +214,10 @@ pub struct ProjectBackfillCursor {
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(default)]
pub struct EmailBackfillRequest {
+ /// Reindex exactly these threads, ignoring `since`. Pages over the given
+ /// ids rather than scanning the table, so a targeted repair costs one
+ /// primary-key lookup per page.
+ pub thread_ids: Vec<uuid::Uuid>,
/// Only backfill threads updated at or after this instant.
pub since: Option<DateTime<Utc>>,
pub index_override: Option<String>,
diff --git a/services/search_processing_service/src/outbound/source.rs b/services/search_processing_service/src/outbound/source.rs
index c2c77cab4b..24476efd96 100644
--- a/services/search_processing_service/src/outbound/source.rs
+++ b/services/search_processing_service/src/outbound/source.rs
@@ -280,6 +280,29 @@ impl BackfillSource for PgBackfillSource {
.filter(|n| *n > 0)
.unwrap_or(DEFAULT_EMAIL_BATCH_SIZE);
+ // An explicit id list pages in memory: each page is a primary-key
+ // lookup, so a targeted repair never runs the scan-and-sort that the
+ // `since` and full variants depend on.
+ if !req.thread_ids.is_empty() {
+ let page = page_of(&req.thread_ids, offset, self.page_sizes.emails);
+ if page.is_empty() {
+ return Ok(SourcePage::empty());
+ }
+ let rows = email_db_client::threads::get::get_thread_ids_with_macro_user_id_by_ids(
+ &self.db, page,
+ )
+ .await
+ .map_err(BackfillError::Source)?;
+ // Advance by the ids consumed, not the rows found, or unknown ids
+ // would stall the drain loop on the same page forever.
+ return Ok(email_source_page(
+ rows,
+ batch_size,
+ page.len(),
+ req.index_override.as_deref(),
+ ));
+ }
+
let rows = match req.since {
Some(since) => {
email_db_client::threads::get::get_paginated_thread_ids_with_macro_user_id_since(
@@ -305,34 +328,12 @@ impl BackfillSource for PgBackfillSource {
return Ok(SourcePage::empty());
}
- let mut by_user: HashMap<String, Vec<String>> = HashMap::new();
- for (thread_id, macro_user_id) in rows {
- by_user
- .entry(macro_user_id)
- .or_default()
- .push(thread_id.to_string());
- }
-
- let messages: Vec<SearchQueueMessage> = by_user
- .into_iter()
- .flat_map(|(macro_user_id, thread_ids)| {
- thread_ids
- .chunks(batch_size)
- .map(|chunk| {
- SearchQueueMessage::ExtractEmailThreadBatch(EmailThreadBatchMessage {
- thread_ids: chunk.to_vec(),
- macro_user_id: macro_user_id.clone(),
- index_override: req.index_override.clone(),
- })
- })
- .collect::<Vec<_>>()
- })
- .collect();
-
- Ok(SourcePage {
- messages,
+ Ok(email_source_page(
+ rows,
+ batch_size,
rows_consumed,
- })
+ req.index_override.as_deref(),
+ ))
}
async fn fetch_entity_properties(
@@ -408,3 +409,52 @@ impl BackfillSource for PgBackfillSource {
))
}
}
+
+/// The `offset..offset + limit` window of `ids`, empty once the offset passes
+/// the end so the drain loop terminates.
+fn page_of(ids: &[uuid::Uuid], offset: usize, limit: usize) -> &[uuid::Uuid] {
+ let start = offset.min(ids.len());
+ let end = start.saturating_add(limit).min(ids.len());
+ &ids[start..end]
+}
+
+/// Group resolved threads by owner and chunk each owner's threads into batch
+/// messages. `rows_consumed` is what the drain loop advances its offset by.
+fn email_source_page(
+ rows: Vec<(uuid::Uuid, String)>,
+ batch_size: usize,
+ rows_consumed: usize,
+ index_override: Option<&str>,
+) -> SourcePage {
+ let mut by_user: HashMap<String, Vec<String>> = HashMap::new();
+ for (thread_id, macro_user_id) in rows {
+ by_user
+ .entry(macro_user_id)
+ .or_default()
+ .push(thread_id.to_string());
+ }
+
+ let messages: Vec<SearchQueueMessage> = by_user
+ .into_iter()
+ .flat_map(|(macro_user_id, thread_ids)| {
+ thread_ids
+ .chunks(batch_size)
+ .map(|chunk| {
+ SearchQueueMessage::ExtractEmailThreadBatch(EmailThreadBatchMessage {
+ thread_ids: chunk.to_vec(),
+ macro_user_id: macro_user_id.clone(),
+ index_override: index_override.map(str::to_string),
+ })
+ })
+ .collect::<Vec<_>>()
+ })
+ .collect();
+
+ SourcePage {
+ messages,
+ rows_consumed,
+ }
+}
+
+#[cfg(test)]
+mod test;
diff --git a/services/search_processing_service/src/outbound/source/test.rs b/services/search_processing_service/src/outbound/source/test.rs
new file mode 100644
index 0000000000..4b8344ac1f
--- /dev/null
+++ b/services/search_processing_service/src/outbound/source/test.rs
@@ -0,0 +1,98 @@
+use super::*;
+
+fn uuid(n: u128) -> uuid::Uuid {
+ uuid::Uuid::from_u128(n)
+}
+
+fn thread_ids(page: &SourcePage) -> Vec<Vec<String>> {
+ page.messages
+ .iter()
+ .map(|message| match message {
+ SearchQueueMessage::ExtractEmailThreadBatch(batch) => batch.thread_ids.clone(),
+ other => panic!("unexpected message: {other:?}"),
+ })
+ .collect()
+}
+
+#[test]
+fn page_of_walks_the_id_list() {
+ let ids: Vec<uuid::Uuid> = (0..5).map(uuid).collect();
+
+ assert_eq!(page_of(&ids, 0, 2), &ids[0..2]);
+ assert_eq!(page_of(&ids, 2, 2), &ids[2..4]);
+ assert_eq!(page_of(&ids, 4, 2), &ids[4..5]);
+}
+
+#[test]
+fn page_of_is_empty_past_the_end() {
+ let ids: Vec<uuid::Uuid> = (0..3).map(uuid).collect();
+
+ assert!(page_of(&ids, 3, 10).is_empty());
+ assert!(page_of(&ids, 99, 10).is_empty());
+}
+
+#[test]
+fn page_of_clamps_a_limit_beyond_the_end() {
+ let ids: Vec<uuid::Uuid> = (0..3).map(uuid).collect();
+
+ assert_eq!(page_of(&ids, 0, 100), &ids[..]);
+}
+
+#[test]
+fn groups_threads_by_owner() {
+ let page = email_source_page(
+ vec![
+ (uuid(1), "macro|[email protected]".into()),
+ (uuid(2), "macro|[email protected]".into()),
+ (uuid(3), "macro|[email protected]".into()),
+ ],
+ 50,
+ 3,
+ None,
+ );
+
+ assert_eq!(page.rows_consumed, 3);
+ let mut sizes: Vec<usize> = thread_ids(&page).iter().map(Vec::len).collect();
+ sizes.sort_unstable();
+ assert_eq!(sizes, vec![1, 2]);
+}
+
+#[test]
+fn chunks_each_owner_at_the_batch_size() {
+ let rows: Vec<(uuid::Uuid, String)> = (0..120)
+ .map(|n| (uuid(n), "macro|[email protected]".to_string()))
+ .collect();
+
+ let page = email_source_page(rows, 50, 120, None);
+
+ let mut sizes: Vec<usize> = thread_ids(&page).iter().map(Vec::len).collect();
+ sizes.sort_unstable();
+ assert_eq!(sizes, vec![20, 50, 50]);
+ assert_eq!(page.rows_consumed, 120);
+}
+
+#[test]
+fn rows_consumed_is_independent_of_rows_found() {
+ // Unknown ids resolve to nothing; the loop still has to advance past them.
+ let page = email_source_page(vec![(uuid(1), "macro|[email protected]".into())], 50, 10, None);
+
+ assert_eq!(page.rows_consumed, 10);
+ assert_eq!(thread_ids(&page), vec![vec![uuid(1).to_string()]]);
+}
+
+#[test]
+fn carries_the_index_override() {
+ let page = email_source_page(
+ vec![(uuid(1), "macro|[email protected]".into())],
+ 50,
+ 1,
+ Some("emails_v2"),
+ );
+
+ match &page.messages[0] {
+ SearchQueueMessage::ExtractEmailThreadBatch(batch) => {
+ assert_eq!(batch.index_override.as_deref(), Some("emails_v2"));
+ }
+ other => panic!("unexpected message: {other:?}"),
+ }
+}