Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(email): connect users to their whole Google address book

Jacob Beckermanbdf06b016d5e
diff --git a/services/email_service/src/backfill_init_service.rs b/services/email_service/src/backfill_init_service.rs
index 1bdb81ad8e..eaaab7d61a 100644
--- a/services/email_service/src/backfill_init_service.rs
+++ b/services/email_service/src/backfill_init_service.rs
@@ -91,6 +91,7 @@ async fn initialize_backfill(
&ctx.email_api,
&ctx.sqs_client,
&ctx.macro_event_broker,
+ &*ctx.contacts_ingress,
)
.await
{
diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
index 94c1744292..4499d79e24 100644
--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
@@ -728,6 +728,7 @@ async fn main() -> anyhow::Result<()> {
let connection_gateway_client_link_manager = connection_gateway_client.clone();
let notification_ingress_service_link_manager = notification_ingress_service.clone();
let macro_event_broker_link_manager = macro_event_broker.clone();
+ let contacts_ingress_link_manager = contacts_ingress.clone();
let cancellation_token = worker_cancellation_token.clone();
// daily link_manager operations for user contacts and inbox subscriptions
worker_tracker.spawn(async move {
@@ -742,6 +743,7 @@ async fn main() -> anyhow::Result<()> {
connection_gateway_client_link_manager,
notification_ingress_service_link_manager,
macro_event_broker_link_manager,
+ contacts_ingress_link_manager,
cancellation_token,
)
.await;
diff --git a/services/email_service/src/composition.rs b/services/email_service/src/composition.rs
index 4c59ffad98..5be460b72f 100644
--- a/services/email_service/src/composition.rs
+++ b/services/email_service/src/composition.rs
@@ -4,6 +4,7 @@ use crate::outbound::{
email_api::{GmailApi, MicrosoftCredentialsClient},
inbox_health::{PgInboxHealth, ProviderInboxHealth},
};
+use contacts::{domain::service::SqsContactsIngress, outbound::ingress::SqsContactsQueue};
use email::{
domain::mailbox::{credentials::MailboxCredentials, health::InboxHealthService},
outbound::mailbox_pg::PgMailboxSync,
@@ -36,7 +37,11 @@ pub fn inbox_lifecycle(
pub type MicrosoftTokens = MailboxCredentials<PgMailboxSync, MicrosoftCredentialsClient>;
pub type InboxHealth = InboxHealthService<
PgInboxHealth,
- ProviderInboxHealth<MicrosoftTokens, MacroEventBrokerService<KafkaEventPublisher, TaskTracker>>,
+ ProviderInboxHealth<
+ MicrosoftTokens,
+ MacroEventBrokerService<KafkaEventPublisher, TaskTracker>,
+ SqsContactsIngress<SqsContactsQueue>,
+ >,
>;
pub fn inbox_health(
@@ -46,6 +51,7 @@ pub fn inbox_health(
redis: crate::util::redis::RedisClient,
queue: sqs_client::SQS,
broker: MacroEventBrokerService<KafkaEventPublisher, TaskTracker>,
+ contacts: std::sync::Arc<SqsContactsIngress<SqsContactsQueue>>,
) -> InboxHealth {
InboxHealthService::new(
PgInboxHealth {
@@ -62,6 +68,7 @@ pub fn inbox_health(
db,
queue,
broker,
+ contacts,
},
)
}
diff --git a/services/email_service/src/outbound/inbox_health.rs b/services/email_service/src/outbound/inbox_health.rs
index 5f96cc2413..0572af2d31 100644
--- a/services/email_service/src/outbound/inbox_health.rs
+++ b/services/email_service/src/outbound/inbox_health.rs
@@ -1,3 +1,4 @@
+use contacts::domain::ports::ContactsIngress;
use email::domain::mailbox::{MailboxError, health::*};
use email_api_client::domain::{
models::{AccessToken, EmailApiError, MailboxAccess, TokenError, TokenFreshness},
@@ -74,14 +75,17 @@ impl InboxHealthRepository for PgInboxHealth {
}
}
-pub struct ProviderInboxHealth<T, B> {
+pub struct ProviderInboxHealth<T, B, C> {
pub gmail: crate::outbound::email_api::GmailApi,
pub outlook: T,
pub db: sqlx::PgPool,
pub queue: sqs_client::SQS,
pub broker: B,
+ pub contacts: std::sync::Arc<C>,
}
-impl<T: MailboxTokenSource, B: MacroEventBroker> InboxHealthGateway for ProviderInboxHealth<T, B> {
+impl<T: MailboxTokenSource, B: MacroEventBroker, C: ContactsIngress> InboxHealthGateway
+ for ProviderInboxHealth<T, B, C>
+{
async fn acquire(
&self,
binding: &InboxHealthBinding,
@@ -128,6 +132,7 @@ impl<T: MailboxTokenSource, B: MacroEventBroker> InboxHealthGateway for Provider
&self.gmail,
&self.queue,
&self.broker,
+ &*self.contacts,
)
.await
.map_err(|_| MailboxError::Persistence)
diff --git a/services/email_service/src/pubsub/link_manager/worker.rs b/services/email_service/src/pubsub/link_manager/worker.rs
index 4faa52d584..6b634b233a 100644
--- a/services/email_service/src/pubsub/link_manager/worker.rs
+++ b/services/email_service/src/pubsub/link_manager/worker.rs
@@ -6,6 +6,7 @@ use crate::pubsub::worker_lifecycle::run_until_cancelled;
use crate::util::redis::RedisClient;
use authentication_service_client::AuthServiceClient;
use connection_gateway_client::client::ConnectionGatewayClient;
+use contacts::{domain::service::SqsContactsIngress, outbound::ingress::SqsContactsQueue};
use futures::StreamExt;
use sqlx::PgPool;
use sqs_client::SQS;
@@ -25,6 +26,7 @@ pub async fn run_worker(
connection_gateway_client: ConnectionGatewayClient,
notification_ingress_service: Arc<NotificationIngressType>,
macro_event_broker: PubSubEventBroker,
+ contacts_ingress: Arc<SqsContactsIngress<SqsContactsQueue>>,
) {
run_worker_with_cancellation(
worker,
@@ -37,6 +39,7 @@ pub async fn run_worker(
connection_gateway_client,
notification_ingress_service,
macro_event_broker,
+ contacts_ingress,
CancellationToken::new(),
)
.await;
@@ -57,6 +60,7 @@ pub async fn run_worker_with_cancellation(
connection_gateway_client: ConnectionGatewayClient,
notification_ingress_service: Arc<NotificationIngressType>,
macro_event_broker: PubSubEventBroker,
+ contacts_ingress: Arc<SqsContactsIngress<SqsContactsQueue>>,
cancellation_token: CancellationToken,
) {
let outlook = loop {
@@ -104,6 +108,7 @@ pub async fn run_worker_with_cancellation(
redis_client.clone(),
sqs_client.clone(),
macro_event_broker.clone(),
+ contacts_ingress,
)),
db,
sqs_worker: worker.clone(),
diff --git a/services/email_service/src/server.rs b/services/email_service/src/server.rs
index d2cde19402..c6b4059918 100644
--- a/services/email_service/src/server.rs
+++ b/services/email_service/src/server.rs
@@ -317,6 +317,12 @@ pub async fn run() -> anyhow::Result<()> {
redis_client.as_ref().clone(),
sqs_client.as_ref().clone(),
macro_event_broker.clone(),
+ std::sync::Arc::new(contacts::domain::service::SqsContactsIngress {
+ queue: contacts::outbound::ingress::SqsContactsQueue::new(
+ aws_sdk_sqs::Client::new(&aws_config),
+ macro_queues::ContactsQueue::new().to_string(),
+ ),
+ }),
)),
attachment_reads: Arc::new(email::domain::attachment_access::AttachmentReadService {
repository: EmailPgRepo::new(db.clone()),
diff --git a/services/email_service/src/util/sync_contacts.rs b/services/email_service/src/util/sync_contacts.rs
index 78554236dc..e1437d9b7e 100644
--- a/services/email_service/src/util/sync_contacts.rs
+++ b/services/email_service/src/util/sync_contacts.rs
@@ -1,6 +1,7 @@
use crate::outbound::email_api::GmailApi;
use crate::pubsub::util::publish_email_event;
use anyhow::{Context, anyhow};
+use contacts::domain::{models::messages::ContactConnection, ports::ContactsIngress};
use email::domain::events::{
EmailMacroEvent, ThreadsReindexReason, ThreadsReindexRequestedMetadata,
};
@@ -22,12 +23,13 @@ use uuid::Uuid;
mod test;
/// Syncs user's contacts with gmail
-pub async fn sync_contacts<B: MacroEventBroker>(
+pub async fn sync_contacts<B: MacroEventBroker, I: ContactsIngress>(
link: &Link,
db: &PgPool,
email_api: &GmailApi,
sqs_client: &SQS,
macro_event_broker: &B,
+ contacts_ingress: &I,
) -> anyhow::Result<()> {
// 1. Get existing sync tokens from our DB
let (contacts_sync_token, other_contacts_sync_token) =
@@ -44,6 +46,9 @@ pub async fn sync_contacts<B: MacroEventBroker>(
// 3. If we received any new/updated contacts, process and store them.
if !new_contacts.is_empty() {
+ // Connect the user to their address book before storing it, so a failed
+ // enqueue leaves the sync tokens untouched and the next sync retries.
+ connect_address_book(contacts_ingress, link, &new_contacts).await?;
Box::pin(process_and_store_contacts(
db,
sqs_client,
@@ -62,6 +67,60 @@ pub async fn sync_contacts<B: MacroEventBroker>(
Ok(())
}
+/// How many address-book connections go in one contacts message.
+const CONNECTIONS_PER_MESSAGE: usize = 500;
+
+/// The connections from the link's owner to each distinct, non-automated
+/// address in `contacts`.
+fn address_book_connections(link: &Link, contacts: &[Contact]) -> Vec<ContactConnection> {
+ let own_email = link.email_address.0.as_ref().to_ascii_lowercase();
+ let mut seen: HashSet<String> = HashSet::new();
+ contacts
+ .iter()
+ .filter_map(|contact| contact.email_address.as_deref())
+ .map(|email| email.trim().to_ascii_lowercase())
+ .filter(|email| {
+ !email.is_empty()
+ && *email != own_email
+ && !email_utils::is_generic_email(email)
+ && seen.insert(email.clone())
+ })
+ .filter_map(|email| match MacroUserIdStr::try_from_email(&email) {
+ Ok(contact) => Some(ContactConnection::new(link.macro_id.clone(), contact)),
+ Err(error) => {
+ // One malformed address must not drop the rest of the address book.
+ tracing::warn!(error = ?error, "skipping invalid address-book email");
+ None
+ }
+ })
+ .collect()
+}
+
+/// Connects the link's owner to everyone in their synced address book, so the
+/// people they know, not only the people they have emailed, show up wherever
+/// Macro suggests or @mentions a contact.
+async fn connect_address_book<I: ContactsIngress>(
+ contacts_ingress: &I,
+ link: &Link,
+ contacts: &[Contact],
+) -> anyhow::Result<()> {
+ if cfg!(not(feature = "contacts_sync")) {
+ return Ok(());
+ }
+ for chunk in address_book_connections(link, contacts).chunks(CONNECTIONS_PER_MESSAGE) {
+ contacts_ingress
+ .enqueue_contact_connections(chunk.to_vec())
+ .await
+ .map_err(|error| {
+ anyhow!("{error:?}").context(format!(
+ "Unable to connect address book contacts for link_id: {}",
+ link.id
+ ))
+ })?;
+ }
+ Ok(())
+}
+
/// Retrieves the current contact and other_contact sync tokens from the database for a given link.
async fn fetch_existing_sync_tokens(
db: &PgPool,
diff --git a/services/email_service/src/util/sync_contacts/test.rs b/services/email_service/src/util/sync_contacts/test.rs
index fd215499dc..7cd6032d35 100644
--- a/services/email_service/src/util/sync_contacts/test.rs
+++ b/services/email_service/src/util/sync_contacts/test.rs
@@ -183,3 +183,51 @@ async fn publishes_a_second_batch_for_fifty_one_threads() {
async fn publishes_ordered_bounded_batches_for_larger_inputs() {
assert_published_batches(137, &[50, 50, 37]);
}
+
+fn link_for(macro_id: &str, email: &str) -> Link {
+ let macro_id = MacroUserIdStr::try_from(macro_id.to_string()).expect("valid macro id");
+ let email_address = macro_user_id::email::EmailStr::try_from(email.to_string()).expect("email");
+ Link {
+ id: Uuid::nil(),
+ is_primary: Link::derive_is_primary(&macro_id, &email_address),
+ macro_id,
+ fusionauth_user_id: "fa".to_string(),
+ email_address,
+ provider: models_email::service::link::UserProvider::Gmail,
+ is_sync_active: true,
+ needs_reauth: false,
+ last_sync_error_at: None,
+ created_at: chrono::Utc::now(),
+ updated_at: chrono::Utc::now(),
+ }
+}
+
+#[test]
+fn address_book_connections_cover_saved_contacts_and_skip_noise() {
+ let link = link_for("macro|[email protected]", "[email protected]");
+ let contacts = vec![
+ other_contact(Some("[email protected]"), Some("Alice")),
+ other_contact(Some("[email protected]"), Some("Alice again")),
+ other_contact(Some("[email protected]"), Some("Me")),
+ other_contact(Some("[email protected]"), None),
+ other_contact(Some(" "), None),
+ other_contact(None, Some("No address")),
+ other_contact(Some("[email protected]"), Some("Friend")),
+ ];
+
+ let connections = address_book_connections(&link, &contacts);
+
+ let second: Vec<String> = connections
+ .iter()
+ .map(|connection| connection.second.to_string())
+ .collect();
+ assert_eq!(
+ second,
+ vec!["macro|[email protected]", "macro|[email protected]"]
+ );
+ assert!(
+ connections
+ .iter()
+ .all(|connection| connection.first == link.macro_id)
+ );
+}