Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(document-text-extractor): skip unopenable PDFs and deleted documents instead of retrying (#7902)
teo140f15ef9651
diff --git a/Cargo.lock b/Cargo.lockindex 1748cd7671..34af210470 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -6424,6 +6424,7 @@ dependencies = ["chrono","lambda_runtime","macro_aws_config",+ "macro_db_migrator","macro_entrypoint","macro_env_var","mockall",diff --git a/services/document_text_extractor/Cargo.toml b/services/document_text_extractor/Cargo.tomlindex 7c2fa0f811..330d686857 100644--- a/services/document_text_extractor/Cargo.toml+++ b/services/document_text_extractor/Cargo.toml@@ -37,3 +37,5 @@ urlencoding = { workspace = true }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }+[dev-dependencies]+macro_db_migrator = { path = "../../crates/macro_db_migrator" }diff --git a/services/document_text_extractor/fixtures/document_text.sql b/services/document_text_extractor/fixtures/document_text.sqlnew file mode 100644index 0000000000..e17717bb98--- /dev/null+++ b/services/document_text_extractor/fixtures/document_text.sql@@ -0,0 +1,8 @@+INSERT INTO public."macro_user" ("id", "username", "email", "stripe_customer_id")+VALUES ('a1111111-1111-1111-1111-111111111111', 'user', '[email protected]', 'stripe_id');++INSERT INTO public."User" ("id", "email", "stripeCustomerId", "macro_user_id")+VALUES ('macro|[email protected]', '[email protected]', 'stripe_id', 'a1111111-1111-1111-1111-111111111111');++INSERT INTO public."Document" ("id", "name", "fileType", "owner")+VALUES ('document-one', 'document', 'docx', 'macro|[email protected]');diff --git a/services/document_text_extractor/src/handler/extract_text_citations.rs b/services/document_text_extractor/src/handler/extract_text_citations.rsindex d3481fa9b5..d6f99ed85f 100644--- a/services/document_text_extractor/src/handler/extract_text_citations.rs+++ b/services/document_text_extractor/src/handler/extract_text_citations.rs@@ -7,7 +7,7 @@ use uuid::Uuid;use crate::{model::key::DocumentKey,- service::{self},+ service::{self, db::DocumentTextOutcome},};use anyhow::{Context, Result};@@ -203,6 +203,13 @@ pub async fn extract_text_from_document(tracing::warn!("ran into encrypted pdf, skipping");return Ok(None);}+ // corrupt or unsupported pdfs fail the same way on every retry+ Err(PdfiumError::PdfiumLibraryInternalError(+ error @ (PdfiumInternalError::FormatError | PdfiumInternalError::SecurityError),+ )) => {+ tracing::warn!(error=?error, "pdfium cannot open pdf, skipping");+ return Ok(None);+ }Err(e) => {tracing::error!(error=?e, "pdfium failed to load pdf from bytes");return Err(Error::from(e));@@ -217,9 +224,17 @@ pub async fn extract_text_from_document(Error::from(e)})?;- db.create_document_text(document_id, &extracted_text, token_count)+ match db+ .create_document_text(document_id, &extracted_text, token_count).await- .context("unable to store extracted text in db")?;+ .context("unable to store extracted text in db")?+ {+ DocumentTextOutcome::Stored => {}+ DocumentTextOutcome::DocumentMissing => {+ tracing::warn!(document_id=%document_id, "document no longer exists, skipping");+ return Ok(None);+ }+ }db.insert_references(&references, document_id).awaitdiff --git a/services/document_text_extractor/src/service/db.rs b/services/document_text_extractor/src/service/db.rsindex 3cec34a805..a9503cddab 100644--- a/services/document_text_extractor/src/service/db.rs+++ b/services/document_text_extractor/src/service/db.rs@@ -15,6 +15,15 @@ pub use MockDBClient as DB;use model::citations::TextReference;+/// Whether extracted text was stored for a document.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DocumentTextOutcome {+ Stored,+ /// The document row no longer exists, e.g. it was deleted while its+ /// converted PDF was waiting to be extracted.+ DocumentMissing,+}+#[cfg_attr(test, allow(dead_code))]#[derive(Clone)]pub struct DBClient {@@ -34,7 +43,7 @@ impl DBClient {document_id: &str,text: &str,token_count: i64,- ) -> anyhow::Result<()> {+ ) -> anyhow::Result<DocumentTextOutcome> {create_document_text::create_document_text(self.inner.clone(),document_id,diff --git a/services/document_text_extractor/src/service/db/create_document_text.rs b/services/document_text_extractor/src/service/db/create_document_text.rsindex a71cc461d1..60fd30e251 100644--- a/services/document_text_extractor/src/service/db/create_document_text.rs+++ b/services/document_text_extractor/src/service/db/create_document_text.rs@@ -1,15 +1,19 @@+use super::DocumentTextOutcome;use anyhow::Result;use lambda_runtime::tracing;use sqlx::{Pool, Postgres};+#[cfg(test)]+mod test;+#[tracing::instrument(skip(db))]pub async fn create_document_text(db: Pool<Postgres>,document_id: &str,text: &str,token_count: i64,-) -> Result<()> {- sqlx::query_as!(+) -> Result<DocumentTextOutcome> {+ let inserted = sqlx::query_as!(DocumentText,r#"INSERT INTO "DocumentText" ("documentId", "content", "tokenCount")@@ -22,7 +26,17 @@ pub async fn create_document_text(token_count).execute(&db)- .await?;+ .await;- Ok(())+ match inserted {+ Ok(_) => Ok(DocumentTextOutcome::Stored),+ // "DocumentText" has a single foreign key, to "Document"+ Err(e)+ if e.as_database_error()+ .is_some_and(|e| e.is_foreign_key_violation()) =>+ {+ Ok(DocumentTextOutcome::DocumentMissing)+ }+ Err(e) => Err(e.into()),+ }}diff --git a/services/document_text_extractor/src/service/db/create_document_text/test.rs b/services/document_text_extractor/src/service/db/create_document_text/test.rsnew file mode 100644index 0000000000..7d9c8340f0--- /dev/null+++ b/services/document_text_extractor/src/service/db/create_document_text/test.rs@@ -0,0 +1,23 @@+use super::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;++async fn check(pool: &Pool<Postgres>, document_id: &str, expected: DocumentTextOutcome) {+ let outcome = create_document_text(pool.clone(), document_id, "extracted text", 2)+ .await+ .unwrap();+ assert_eq!(outcome, expected, "{document_id}");+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("document_text"))+)]+async fn text_for_a_deleted_document_is_reported_missing_instead_of_failing(pool: Pool<Postgres>) {+ check(&pool, "document-one", DocumentTextOutcome::Stored).await;+ check(+ &pool,+ "deleted-document",+ DocumentTextOutcome::DocumentMissing,+ )+ .await;+}