Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
#[cfg(test)]
pub(crate) mod test;
use crate::attachments::provider;
use crate::messages::replying_to_id;
use crate::parse::service_to_db::addresses_from_message;
use crate::{contacts, labels, messages, parse, threads};
use models_email::email::db::address::UpsertedRecipients;
use models_email::email::service::label::system_labels;
use models_email::email::service::message;
use sqlx::PgPool;
use sqlx::types::Uuid;
/// What a re-sync of a stored message read before fetching it from Gmail.
#[derive(Debug, Clone, Default)]
pub struct ResyncSnapshot {
/// The message's provider labels before the fetch.
pub stored_labels: Vec<String>,
/// Labels with a pending Macro write just before or just after the fetch.
pub pending_labels: Vec<String>,
}
/// inserts a message (and all its related parts) into the database using the passed transaction.
pub async fn insert_message_with_tx(
tx: &mut sqlx::PgConnection,
thread_db_id: Uuid,
message: &mut message::Message,
link_id: Uuid,
// addresses (and labels) need to be inserted ahead of time outside the tx, as they are shared
// across messages and can cause deadlocks if inserted within.
recipents: UpsertedRecipients,
// determines whether to update thread metadata (inbox_visible, timestamps). set to false when:
// 1. inserting thread simultaneously (thread already has latest values)
// 2. backfilling messages (metadata gets updated once after all messages complete)
update_thread_metadata: bool,
// Set when re-syncing a stored message: Gmail's answer may predate a local
// write made while it was fetched, so labels change only where Gmail
// disagrees with what was stored before the fetch.
resync: Option<&ResyncSnapshot>,
) -> anyhow::Result<()> {
if let Some(provider_thread_id) = message.provider_thread_id.as_deref() {
threads::provider_identity::lock_provider_thread(tx, link_id, provider_thread_id).await?;
}
if resync.is_some() {
// Before the row upsert, in the same order as label writes take it.
labels::pending_writes::lock_thread_label_writes(tx, &[thread_db_id]).await?;
}
let message_db_id = insert_db_message(
tx,
message,
thread_db_id,
recipents.from_contact_id,
update_thread_metadata,
)
.await?;
contacts::upsert_message::upsert_message_recipients(tx, message_db_id, &recipents).await?;
// Also when Gmail reports no labels, so a re-sync removes the stored ones.
let provider_label_ids: Vec<String> = message
.labels
.iter()
.map(|label| label.provider_label_id.clone())
.collect();
// A re-sync must not undo a Macro label write Gmail has not confirmed yet,
// including the read and starred flags those labels drive.
let mut kept_label_ids =
labels::pending_writes::pending_provider_label_ids(&mut *tx, message_db_id).await?;
if let Some(snapshot) = resync {
kept_label_ids.extend(snapshot.pending_labels.iter().cloned());
let stored_now = sqlx::query_scalar!(
r#"
SELECT l.provider_label_id
FROM email_message_labels ml
JOIN email_labels l ON l.id = ml.label_id
WHERE ml.message_id = $1
"#,
message_db_id,
)
.fetch_all(&mut *tx)
.await?;
kept_label_ids.extend(
stored_now
.into_iter()
.chain(provider_label_ids.iter().cloned())
.chain(snapshot.stored_labels.iter().cloned())
.filter(|label| {
provider_label_ids.contains(label) == snapshot.stored_labels.contains(label)
}),
);
}
kept_label_ids.sort_unstable();
kept_label_ids.dedup();
labels::insert::insert_message_labels(
tx,
link_id,
message_db_id,
&provider_label_ids,
true,
&kept_label_ids,
)
.await?;
let kept_read = kept_label_ids.iter().any(|id| id == system_labels::UNREAD);
let kept_starred = kept_label_ids.iter().any(|id| id == system_labels::STARRED);
if kept_read || kept_starred {
messages::update::set_flags_from_labels(tx, message_db_id, kept_read, kept_starred).await?;
}
provider::insert_attachments(tx, message_db_id, &mut message.attachments).await?;
if update_thread_metadata {
threads::update::update_thread_metadata(tx, thread_db_id, link_id).await?;
replying_to_id::update_message_replying_to_from_headers(
tx,
message,
message_db_id,
link_id,
)
.await?;
}
Ok(())
}
/// inserts message object into the database
///
/// On conflict the update is skipped entirely when the stored row already
/// holds the incoming values, so a re-sync writes no new row version. When
/// only small columns change, the large body/header columns keep their
/// existing values, which lets Postgres reuse their TOAST chunks instead of
/// rewriting them. Provider messages arrive without `replying_to_id` (it is
/// resolved from headers afterwards), so a NULL keeps the stored value.
#[tracing::instrument(skip(tx, message), err)]
async fn insert_db_message(
tx: &mut sqlx::PgConnection,
message: &mut message::Message,
thread_id: Uuid,
from_contact_id: Option<Uuid>,
update_thread_metadata: bool,
) -> anyhow::Result<Uuid> {
let db_message =
parse::service_to_db::map_service_message_to_db(message, thread_id, from_contact_id);
let result = sqlx::query_scalar!(
r#"
INSERT INTO email_messages (
id, provider_id, link_id, global_id, thread_id, provider_thread_id, replying_to_id, provider_history_id, internal_date_ts,
snippet, size_estimate, subject, from_name, from_contact_id, sent_at, has_attachments, is_read,
is_starred, is_sent, is_draft, body_text, body_html_sanitized, headers_jsonb
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23)
ON CONFLICT (link_id, provider_id) WHERE provider_id IS NOT NULL DO UPDATE SET
global_id = EXCLUDED.global_id,
provider_history_id = EXCLUDED.provider_history_id,
provider_thread_id = EXCLUDED.provider_thread_id,
replying_to_id = COALESCE(EXCLUDED.replying_to_id, email_messages.replying_to_id),
from_name = EXCLUDED.from_name,
from_contact_id = EXCLUDED.from_contact_id,
internal_date_ts = EXCLUDED.internal_date_ts,
snippet = EXCLUDED.snippet,
size_estimate = EXCLUDED.size_estimate,
subject = EXCLUDED.subject,
sent_at = EXCLUDED.sent_at,
has_attachments = EXCLUDED.has_attachments,
is_read = EXCLUDED.is_read,
is_starred = EXCLUDED.is_starred,
is_sent = EXCLUDED.is_sent,
is_draft = EXCLUDED.is_draft,
headers_jsonb = CASE
WHEN email_messages.headers_jsonb IS DISTINCT FROM EXCLUDED.headers_jsonb
THEN EXCLUDED.headers_jsonb ELSE email_messages.headers_jsonb END,
body_text = CASE
WHEN email_messages.body_text IS DISTINCT FROM EXCLUDED.body_text
THEN EXCLUDED.body_text ELSE email_messages.body_text END,
body_html_sanitized = CASE
WHEN email_messages.body_html_sanitized IS DISTINCT FROM EXCLUDED.body_html_sanitized
THEN EXCLUDED.body_html_sanitized ELSE email_messages.body_html_sanitized END,
updated_at = NOW()
-- Large columns last: the row comparison stops at the first difference.
WHERE (
email_messages.global_id, email_messages.provider_history_id,
email_messages.provider_thread_id, email_messages.replying_to_id,
email_messages.from_name, email_messages.from_contact_id,
email_messages.internal_date_ts, email_messages.snippet,
email_messages.size_estimate, email_messages.subject, email_messages.sent_at,
email_messages.has_attachments, email_messages.is_read, email_messages.is_starred,
email_messages.is_sent, email_messages.is_draft, email_messages.headers_jsonb,
email_messages.body_text, email_messages.body_html_sanitized
) IS DISTINCT FROM (
EXCLUDED.global_id, EXCLUDED.provider_history_id,
EXCLUDED.provider_thread_id, COALESCE(EXCLUDED.replying_to_id, email_messages.replying_to_id),
EXCLUDED.from_name, EXCLUDED.from_contact_id,
EXCLUDED.internal_date_ts, EXCLUDED.snippet,
EXCLUDED.size_estimate, EXCLUDED.subject, EXCLUDED.sent_at,
EXCLUDED.has_attachments, EXCLUDED.is_read, EXCLUDED.is_starred,
EXCLUDED.is_sent, EXCLUDED.is_draft, EXCLUDED.headers_jsonb,
EXCLUDED.body_text, EXCLUDED.body_html_sanitized
)
RETURNING id
"#,
db_message.id,
db_message.provider_id,
db_message.link_id,
db_message.global_id,
thread_id,
db_message.provider_thread_id,
db_message.replying_to_id,
db_message.provider_history_id,
db_message.internal_date_ts,
db_message.snippet,
db_message.size_estimate,
db_message.subject,
db_message.from_name,
db_message.from_contact_id,
db_message.sent_at,
db_message.has_attachments,
db_message.is_read,
db_message.is_starred,
db_message.is_sent,
db_message.is_draft,
db_message.body_text,
db_message.body_html_sanitized,
db_message.headers_jsonb
)
.fetch_optional(&mut *tx)
.await?;
if let Some(id) = result {
return Ok(id);
}
// The conflicting row already holds this data; fetch its id.
let existing_id = sqlx::query_scalar!(
r#"
SELECT id FROM email_messages
WHERE link_id = $1 AND provider_id = $2
"#,
db_message.link_id,
db_message.provider_id,
)
.fetch_one(&mut *tx)
.await?;
Ok(existing_id)
}
/// A message a backfill write left out because its own data can never be
/// stored; the rest of its thread was still written.
#[derive(Debug)]
pub struct SkippedMessage {
/// The message's position in the slice passed to the write.
pub index: usize,
/// Why it could not be stored.
pub error: anyhow::Error,
}
/// Writes backfilled messages one savepoint at a time.
///
/// A message whose contacts or rows fail with an error `is_unstorable`
/// accepts is skipped and reported, and the others still commit: one bad
/// message must not cost its whole thread. Any other error fails the write.
pub(crate) struct BackfillMessageWriter<'a, U> {
pub(crate) pool: &'a PgPool,
pub(crate) link_id: Uuid,
pub(crate) is_unstorable: U,
}
impl<U: Fn(&anyhow::Error) -> bool> BackfillMessageWriter<'_, U> {
/// Upserts each message's contacts, in order. Contacts are shared across
/// messages, so this runs before the messages' transaction to avoid
/// deadlocks. A skipped message gets `None`.
pub(crate) async fn upsert_contacts(
&self,
messages: &[message::Message],
skipped: &mut Vec<SkippedMessage>,
) -> anyhow::Result<Vec<Option<UpsertedRecipients>>> {
let mut recipients = Vec::with_capacity(messages.len());
for (index, message) in messages.iter().enumerate() {
match contacts::upsert_message::parse_and_upsert_message_contacts(
self.pool,
self.link_id,
addresses_from_message(message),
)
.await
{
Ok(upserted) => recipients.push(Some(upserted)),
Err(error) if (self.is_unstorable)(&error) => {
skipped.push(SkippedMessage { index, error });
recipients.push(None);
}
Err(error) => return Err(error),
}
}
Ok(recipients)
}
/// Inserts each message whose contacts were stored, under its own savepoint.
pub(crate) async fn insert_messages(
&self,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
thread_db_id: Uuid,
messages: &mut [message::Message],
recipients: Vec<Option<UpsertedRecipients>>,
skipped: &mut Vec<SkippedMessage>,
) -> anyhow::Result<()> {
for (index, (message, recipients)) in messages.iter_mut().zip(recipients).enumerate() {
let Some(recipients) = recipients else {
continue;
};
let mut savepoint = sqlx::Acquire::begin(&mut *tx).await?;
match insert_message_with_tx(
&mut savepoint,
thread_db_id,
message,
self.link_id,
recipients,
false,
None,
)
.await
{
Ok(()) => savepoint.commit().await?,
Err(error) if (self.is_unstorable)(&error) => {
savepoint.rollback().await?;
skipped.push(SkippedMessage { index, error });
}
Err(error) => return Err(error),
}
}
Ok(())
}
}
/// Inserts messages into an existing thread in one transaction, leaving its
/// metadata and `replying_to` links to the caller's later metadata update.
/// Returns the messages it skipped because `is_unstorable` accepted their error.
#[tracing::instrument(skip(pool, messages, is_unstorable), fields(message_count = messages.len()), err)]
pub async fn insert_thread_messages(
pool: &PgPool,
thread_db_id: Uuid,
link_id: Uuid,
messages: &mut [message::Message],
is_unstorable: impl Fn(&anyhow::Error) -> bool,
) -> anyhow::Result<Vec<SkippedMessage>> {
let writer = BackfillMessageWriter {
pool,
link_id,
is_unstorable,
};
let mut skipped = Vec::new();
let recipients = writer.upsert_contacts(messages, &mut skipped).await?;
let mut tx = pool.begin().await?;
writer
.insert_messages(&mut tx, thread_db_id, messages, recipients, &mut skipped)
.await?;
tx.commit().await?;
Ok(skipped)
}
/// Inserts a single message into the database with transaction handling
#[tracing::instrument(skip(pool, message), fields(link_id = %message.link_id), err)]
pub async fn insert_message(
pool: &PgPool,
thread_id: Uuid,
message: &mut message::Message,
link_id: Uuid,
update_thread_metadata: bool,
resync: Option<&ResyncSnapshot>,
) -> anyhow::Result<()> {
// we have to insert addresses before inserting the message. these values are shared
// across messages, so inserting them in the txn can cause deadlocks.
let addresses = addresses_from_message(message);
let recipients =
contacts::upsert_message::parse_and_upsert_message_contacts(pool, link_id, addresses)
.await?;
let mut tx = pool.begin().await?;
match insert_message_with_tx(
&mut tx,
thread_id,
message,
link_id,
recipients,
update_thread_metadata,
resync,
)
.await
{
Ok(_) => {
tx.commit().await?;
Ok(())
}
Err(e) => {
if let Err(rollback_err) = tx.rollback().await {
return Err(e.context(format!("Rollback also failed: {:?}", rollback_err)));
}
Err(e)
}
}
}