Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(documents): break email-sha advisory lock convoy (#6035)

teoe56ac853a6b5
diff --git a/.sqlx/query-1a543ca6772d6aeb55973a1569dde96d386117e25e82f2b2497249eaa08e0515.json b/.sqlx/query-1a543ca6772d6aeb55973a1569dde96d386117e25e82f2b2497249eaa08e0515.json
new file mode 100644
index 0000000000..0cd3f60e6e
--- /dev/null
+++ b/.sqlx/query-1a543ca6772d6aeb55973a1569dde96d386117e25e82f2b2497249eaa08e0515.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT DISTINCT i.\"documentId\"\n FROM \"DocumentInstance\" i\n WHERE i.sha = $1\n AND (\n SELECT d.owner\n FROM \"Document\" d\n WHERE d.id = i.\"documentId\"\n ) = $2\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "documentId",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "1a543ca6772d6aeb55973a1569dde96d386117e25e82f2b2497249eaa08e0515"
+}
diff --git a/.sqlx/query-5a9a8adfb252636418fcf651c15161fc20de39e58275ad8df06acee434b26cde.json b/.sqlx/query-5a9a8adfb252636418fcf651c15161fc20de39e58275ad8df06acee434b26cde.json
deleted file mode 100644
index 4e58630900..0000000000
--- a/.sqlx/query-5a9a8adfb252636418fcf651c15161fc20de39e58275ad8df06acee434b26cde.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT d.id\n FROM \"Document\" d\n INNER JOIN LATERAL (\n SELECT i.sha\n FROM \"DocumentInstance\" i\n WHERE i.\"documentId\" = d.id\n ORDER BY i.\"createdAt\" DESC\n LIMIT 1\n ) latest ON latest.sha = $2\n WHERE d.owner = $1\n AND d.\"deletedAt\" IS NULL\n AND EXISTS (\n SELECT 1\n FROM document_email de\n WHERE de.document_id = d.id\n )\n ORDER BY d.\"createdAt\" ASC, d.id ASC\n LIMIT 1\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "id",
- "type_info": "Text"
- }
- ],
- "parameters": {
- "Left": [
- "Text",
- "Text"
- ]
- },
- "nullable": [
- false
- ]
- },
- "hash": "5a9a8adfb252636418fcf651c15161fc20de39e58275ad8df06acee434b26cde"
-}
diff --git a/.sqlx/query-8272b8f2a5392c9b47c7f806a68a95ea66a862a2c61e79f118400683c1882e2f.json b/.sqlx/query-8272b8f2a5392c9b47c7f806a68a95ea66a862a2c61e79f118400683c1882e2f.json
new file mode 100644
index 0000000000..79491a7e0e
--- /dev/null
+++ b/.sqlx/query-8272b8f2a5392c9b47c7f806a68a95ea66a862a2c61e79f118400683c1882e2f.json
@@ -0,0 +1,24 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT d.id\n FROM \"Document\" d\n WHERE d.id = ANY($1)\n AND d.owner = $2\n AND d.\"deletedAt\" IS NULL\n AND EXISTS (\n SELECT 1\n FROM document_email de\n WHERE de.document_id = d.id\n )\n AND (\n SELECT i.sha\n FROM \"DocumentInstance\" i\n WHERE i.\"documentId\" = d.id\n ORDER BY i.\"createdAt\" DESC\n LIMIT 1\n ) = $3\n ORDER BY d.\"createdAt\" ASC, d.id ASC\n LIMIT 1\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "8272b8f2a5392c9b47c7f806a68a95ea66a862a2c61e79f118400683c1882e2f"
+}
diff --git a/crates/documents/src/domain/ports.rs b/crates/documents/src/domain/ports.rs
index 0f970c23c5..0819648a51 100644
--- a/crates/documents/src/domain/ports.rs
+++ b/crates/documents/src/domain/ports.rs
@@ -142,10 +142,9 @@ pub trait DocumentRepo: Send + Sync + 'static {
) -> impl Future<Output = Result<DocumentMetadata, Self::Err>> + Send;
/// Import an email attachment: link it to a reusable live email document
- /// owned by the same user, or insert a new document and link it.
- ///
- /// Lookup, possible insert, and the `document_email` write happen in one
- /// transaction under an advisory lock on `(owner, sha)`.
+ /// owned by the same user (matching latest-instance sha), or insert a new
+ /// document and link it. Concurrent first-time creates for the same
+ /// `(owner, sha)` are serialized so two imports cannot insert duplicates.
fn import_email_attachment_document(
&self,
args: ImportEmailAttachmentRepoArgs,
diff --git a/crates/documents/src/outbound/pg_document_repo.rs b/crates/documents/src/outbound/pg_document_repo.rs
index d961038632..d523933091 100644
--- a/crates/documents/src/outbound/pg_document_repo.rs
+++ b/crates/documents/src/outbound/pg_document_repo.rs
@@ -39,6 +39,59 @@ impl PgDocumentRepo {
pub fn new(pool: PgPool) -> Self {
Self { pool }
}
+
+ async fn reused_email_document(
+ &self,
+ document_id: String,
+ ) -> Result<EmailImportRepoOutcome, sqlx::Error> {
+ let metadata = self.get_document_metadata(&document_id).await?;
+ Ok(EmailImportRepoOutcome::Reused(metadata))
+ }
+
+ async fn reused_linked_email_attachment(
+ &self,
+ email_attachment_id: uuid::Uuid,
+ ) -> Result<EmailImportRepoOutcome, sqlx::Error> {
+ let existing_id =
+ create::find_document_id_for_email_attachment(&self.pool, email_attachment_id)
+ .await?
+ .ok_or_else(|| {
+ sqlx::Error::Protocol(
+ "email attachment already linked but document is missing".into(),
+ )
+ })?;
+ self.reused_email_document(existing_id).await
+ }
+
+ async fn find_reusable_email_document_by_sha(
+ &self,
+ owner: &str,
+ sha: &str,
+ ) -> Result<Option<String>, sqlx::Error> {
+ let mut conn = self.pool.acquire().await?;
+ create::find_live_email_document_id_by_sha(&mut conn, owner, sha).await
+ }
+
+ async fn link_existing_email_document(
+ &self,
+ document_id: &str,
+ email_attachment_id: uuid::Uuid,
+ ) -> Result<EmailImportRepoOutcome, sqlx::Error> {
+ let mut transaction = self.pool.begin().await?;
+ match create::link_document_email(&mut transaction, document_id, email_attachment_id).await
+ {
+ Ok(()) => {
+ transaction.commit().await?;
+ self.reused_email_document(document_id.to_string()).await
+ }
+ Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => {
+ transaction.rollback().await?;
+ self.reused_linked_email_attachment(email_attachment_id)
+ .await
+ }
+ Err(e) => Err(e),
+ }
+ }
}
async fn update_document_modified(pool: &PgPool, document_id: &str) -> Result<(), sqlx::Error> {
@@ -437,6 +490,24 @@ impl DocumentRepo for PgDocumentRepo {
create,
} = args;
+ // Unlocked reuse: attachment already linked, or a live email doc with
+ // this sha already exists. The advisory lock is only required when a
+ // concurrent import might insert the first document for (owner, sha).
+ if let Some(existing_id) =
+ create::find_document_id_for_email_attachment(&self.pool, email_attachment_id).await?
+ {
+ return self.reused_email_document(existing_id).await;
+ }
+
+ if let Some(existing_id) = self
+ .find_reusable_email_document_by_sha(create.user_id.as_ref(), &create.sha)
+ .await?
+ {
+ return self
+ .link_existing_email_document(&existing_id, email_attachment_id)
+ .await;
+ }
+
let mut transaction = self.pool.begin().await?;
if let Some(existing_id) = create::reuse_email_document(
@@ -448,8 +519,7 @@ impl DocumentRepo for PgDocumentRepo {
.await?
{
transaction.commit().await?;
- let metadata = self.get_document_metadata(&existing_id).await?;
- return Ok(EmailImportRepoOutcome::Reused(metadata));
+ return self.reused_email_document(existing_id).await;
}
let metadata =
@@ -465,16 +535,9 @@ impl DocumentRepo for PgDocumentRepo {
Ok(()) => {}
Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => {
transaction.rollback().await?;
- let existing_id =
- create::find_document_id_for_email_attachment(&self.pool, email_attachment_id)
- .await?
- .ok_or_else(|| {
- sqlx::Error::Protocol(
- "email attachment already linked but document is missing".into(),
- )
- })?;
- let metadata = self.get_document_metadata(&existing_id).await?;
- return Ok(EmailImportRepoOutcome::Reused(metadata));
+ return self
+ .reused_linked_email_attachment(email_attachment_id)
+ .await;
}
Err(e) => return Err(e),
}
diff --git a/crates/documents/src/outbound/pg_document_repo/create.rs b/crates/documents/src/outbound/pg_document_repo/create.rs
index 7af0852795..95f75a3765 100644
--- a/crates/documents/src/outbound/pg_document_repo/create.rs
+++ b/crates/documents/src/outbound/pg_document_repo/create.rs
@@ -259,39 +259,69 @@ where
.await
}
-pub async fn find_live_email_document_id_by_sha<'e, E>(
- executor: E,
+/// Finds a live email-linked document owned by `owner` whose latest instance
+/// sha matches `sha`.
+///
+/// Split into two queries so Postgres cannot plan this as "all live documents
+/// for the owner, then filter sha". The probe is a sha index scan; owner is
+/// applied as a scalar PK lookup so a `JOIN` / `EXISTS` cannot be rewritten
+/// into an owner-index drive. Reuse is per-owner, so a popular attachment
+/// sha (logo, signature) must not load every tenant's hits into the process.
+/// The second statement applies liveness, `document_email`, and latest-sha.
+#[tracing::instrument(skip(conn), err)]
+pub async fn find_live_email_document_id_by_sha(
+ conn: &mut sqlx::PgConnection,
owner: &str,
sha: &str,
-) -> Result<Option<String>, sqlx::Error>
-where
- E: sqlx::Executor<'e, Database = sqlx::Postgres>,
-{
+) -> Result<Option<String>, sqlx::Error> {
+ let candidate_document_ids = sqlx::query_scalar!(
+ r#"
+ SELECT DISTINCT i."documentId"
+ FROM "DocumentInstance" i
+ WHERE i.sha = $1
+ AND (
+ SELECT d.owner
+ FROM "Document" d
+ WHERE d.id = i."documentId"
+ ) = $2
+ "#,
+ sha,
+ owner,
+ )
+ .fetch_all(&mut *conn)
+ .await?;
+
+ if candidate_document_ids.is_empty() {
+ return Ok(None);
+ }
+
sqlx::query_scalar!(
r#"
SELECT d.id
FROM "Document" d
- INNER JOIN LATERAL (
- SELECT i.sha
- FROM "DocumentInstance" i
- WHERE i."documentId" = d.id
- ORDER BY i."createdAt" DESC
- LIMIT 1
- ) latest ON latest.sha = $2
- WHERE d.owner = $1
+ WHERE d.id = ANY($1)
+ AND d.owner = $2
AND d."deletedAt" IS NULL
AND EXISTS (
SELECT 1
FROM document_email de
WHERE de.document_id = d.id
)
+ AND (
+ SELECT i.sha
+ FROM "DocumentInstance" i
+ WHERE i."documentId" = d.id
+ ORDER BY i."createdAt" DESC
+ LIMIT 1
+ ) = $3
ORDER BY d."createdAt" ASC, d.id ASC
LIMIT 1
"#,
+ &candidate_document_ids,
owner,
sha,
)
- .fetch_optional(executor)
+ .fetch_optional(&mut *conn)
.await
}
@@ -314,6 +344,11 @@ pub async fn link_document_email(
Ok(())
}
+/// Serializes first-time creates of the same `(owner, sha)` email document.
+///
+/// Callers should try unlocked reuse first. This lock is only needed when a
+/// concurrent import might insert the first document for this key.
+#[tracing::instrument(skip(transaction), err)]
pub async fn reuse_email_document(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
owner: &str,
@@ -334,9 +369,7 @@ pub async fn reuse_email_document(
return Ok(Some(document_id));
}
- if let Some(document_id) =
- find_live_email_document_id_by_sha(&mut **transaction, owner, sha).await?
- {
+ if let Some(document_id) = find_live_email_document_id_by_sha(transaction, owner, sha).await? {
link_document_email(transaction, &document_id, email_attachment_id).await?;
return Ok(Some(document_id));
}
diff --git a/crates/documents/src/outbound/pg_document_repo/tests.rs b/crates/documents/src/outbound/pg_document_repo/tests.rs
index 2805b7eb19..4a91b73371 100644
--- a/crates/documents/src/outbound/pg_document_repo/tests.rs
+++ b/crates/documents/src/outbound/pg_document_repo/tests.rs
@@ -2023,3 +2023,56 @@ async fn test_create_document_does_not_reuse_email_document_by_sha(pool: Pool<Po
assert!(matches!(imported, EmailImportRepoOutcome::Created(_)));
assert_ne!(imported.metadata().document_id, created_doc.document_id);
}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("documents_test_data"))
+)]
+async fn test_import_email_attachment_only_reuses_latest_instance_sha(pool: Pool<Postgres>) {
+ let repo = PgDocumentRepo::new(pool.clone());
+ let attachments = insert_email_attachments(&pool, 3).await;
+ let old_sha = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff";
+ let new_sha = "1111111111111111111111111111111111111111111111111111111111111111";
+
+ let first = repo
+ .import_email_attachment_document(
+ import_email_document_args(TEST_DOCUMENT_OWNER_ID, old_sha, attachments[0]),
+ pdf_share_permission(),
+ )
+ .await
+ .unwrap();
+ let document_id = first.metadata().document_id.clone();
+
+ sqlx::query(
+ r#"
+ INSERT INTO "DocumentInstance" ("documentId", sha, "createdAt", "updatedAt")
+ VALUES ($1, $2, NOW() + INTERVAL '1 second', NOW() + INTERVAL '1 second')
+ "#,
+ )
+ .bind(&document_id)
+ .bind(new_sha)
+ .execute(&pool)
+ .await
+ .unwrap();
+
+ let reused_latest = repo
+ .import_email_attachment_document(
+ import_email_document_args(TEST_DOCUMENT_OWNER_ID, new_sha, attachments[1]),
+ pdf_share_permission(),
+ )
+ .await
+ .unwrap();
+ let superseded = repo
+ .import_email_attachment_document(
+ import_email_document_args(TEST_DOCUMENT_OWNER_ID, old_sha, attachments[2]),
+ pdf_share_permission(),
+ )
+ .await
+ .unwrap();
+
+ assert!(matches!(first, EmailImportRepoOutcome::Created(_)));
+ assert!(matches!(reused_latest, EmailImportRepoOutcome::Reused(_)));
+ assert_eq!(document_id, reused_latest.metadata().document_id);
+ assert!(matches!(superseded, EmailImportRepoOutcome::Created(_)));
+ assert_ne!(document_id, superseded.metadata().document_id);
+}
diff --git a/crates/macro_db_client/migrations/20260829020240_document_instance_sha_index.sql b/crates/macro_db_client/migrations/20260829020240_document_instance_sha_index.sql
new file mode 100644
index 0000000000..8d94e36974
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260829020240_document_instance_sha_index.sql
@@ -0,0 +1,7 @@
+-- no-transaction
+-- Email-attachment dedupe looks up DocumentInstance by sha. Without this
+-- index the query has to walk every live document for the owner.
+-- Must stay a single statement: sqlx sends no-transaction migrations as
+-- one batch, and CONCURRENTLY cannot run inside a transaction.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_document_instance_sha
+ ON "DocumentInstance" (sha);
diff --git a/crates/macro_db_client/migrations/20260829020258_document_instance_document_id_created_at_index.sql b/crates/macro_db_client/migrations/20260829020258_document_instance_document_id_created_at_index.sql
new file mode 100644
index 0000000000..5eb5eeb75b
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260829020258_document_instance_document_id_created_at_index.sql
@@ -0,0 +1,6 @@
+-- no-transaction
+-- Latest-instance check after a sha hit: ORDER BY createdAt DESC LIMIT 1
+-- per document. DocumentInstance_documentId_idx cannot satisfy the sort.
+-- Must stay a single statement: see 20260829020240_document_instance_sha_index.sql.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_document_instance_document_id_created_at
+ ON "DocumentInstance" ("documentId", "createdAt" DESC);