Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(document-text-extractor): skip unopenable PDFs and deleted documents instead of retrying (#7902)

teo140f15ef9651
diff --git a/Cargo.lock b/Cargo.lock
index 1748cd7671..34af210470 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6424,6 +6424,7 @@ dependencies = [
"chrono",
"lambda_runtime",
"macro_aws_config",
+ "macro_db_migrator",
"macro_entrypoint",
"macro_env_var",
"mockall",
diff --git a/services/document_text_extractor/Cargo.toml b/services/document_text_extractor/Cargo.toml
index 7c2fa0f811..330d686857 100644
--- a/services/document_text_extractor/Cargo.toml
+++ b/services/document_text_extractor/Cargo.toml
@@ -37,3 +37,5 @@ urlencoding = { workspace = true }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }
+[dev-dependencies]
+macro_db_migrator = { path = "../../crates/macro_db_migrator" }
diff --git a/services/document_text_extractor/fixtures/document_text.sql b/services/document_text_extractor/fixtures/document_text.sql
new file mode 100644
index 0000000000..e17717bb98
--- /dev/null
+++ b/services/document_text_extractor/fixtures/document_text.sql
@@ -0,0 +1,8 @@
+INSERT INTO public."macro_user" ("id", "username", "email", "stripe_customer_id")
+VALUES ('a1111111-1111-1111-1111-111111111111', 'user', '[email protected]', 'stripe_id');
+
+INSERT INTO public."User" ("id", "email", "stripeCustomerId", "macro_user_id")
+VALUES ('macro|[email protected]', '[email protected]', 'stripe_id', 'a1111111-1111-1111-1111-111111111111');
+
+INSERT INTO public."Document" ("id", "name", "fileType", "owner")
+VALUES ('document-one', 'document', 'docx', 'macro|[email protected]');
diff --git a/services/document_text_extractor/src/handler/extract_text_citations.rs b/services/document_text_extractor/src/handler/extract_text_citations.rs
index d3481fa9b5..d6f99ed85f 100644
--- a/services/document_text_extractor/src/handler/extract_text_citations.rs
+++ b/services/document_text_extractor/src/handler/extract_text_citations.rs
@@ -7,7 +7,7 @@ use uuid::Uuid;
use crate::{
model::key::DocumentKey,
- service::{self},
+ service::{self, db::DocumentTextOutcome},
};
use anyhow::{Context, Result};
@@ -203,6 +203,13 @@ pub async fn extract_text_from_document(
tracing::warn!("ran into encrypted pdf, skipping");
return Ok(None);
}
+ // corrupt or unsupported pdfs fail the same way on every retry
+ Err(PdfiumError::PdfiumLibraryInternalError(
+ error @ (PdfiumInternalError::FormatError | PdfiumInternalError::SecurityError),
+ )) => {
+ tracing::warn!(error=?error, "pdfium cannot open pdf, skipping");
+ return Ok(None);
+ }
Err(e) => {
tracing::error!(error=?e, "pdfium failed to load pdf from bytes");
return Err(Error::from(e));
@@ -217,9 +224,17 @@ pub async fn extract_text_from_document(
Error::from(e)
})?;
- db.create_document_text(document_id, &extracted_text, token_count)
+ match db
+ .create_document_text(document_id, &extracted_text, token_count)
.await
- .context("unable to store extracted text in db")?;
+ .context("unable to store extracted text in db")?
+ {
+ DocumentTextOutcome::Stored => {}
+ DocumentTextOutcome::DocumentMissing => {
+ tracing::warn!(document_id=%document_id, "document no longer exists, skipping");
+ return Ok(None);
+ }
+ }
db.insert_references(&references, document_id)
.await
diff --git a/services/document_text_extractor/src/service/db.rs b/services/document_text_extractor/src/service/db.rs
index 3cec34a805..a9503cddab 100644
--- a/services/document_text_extractor/src/service/db.rs
+++ b/services/document_text_extractor/src/service/db.rs
@@ -15,6 +15,15 @@ pub use MockDBClient as DB;
use model::citations::TextReference;
+/// Whether extracted text was stored for a document.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DocumentTextOutcome {
+ Stored,
+ /// The document row no longer exists, e.g. it was deleted while its
+ /// converted PDF was waiting to be extracted.
+ DocumentMissing,
+}
+
#[cfg_attr(test, allow(dead_code))]
#[derive(Clone)]
pub struct DBClient {
@@ -34,7 +43,7 @@ impl DBClient {
document_id: &str,
text: &str,
token_count: i64,
- ) -> anyhow::Result<()> {
+ ) -> anyhow::Result<DocumentTextOutcome> {
create_document_text::create_document_text(
self.inner.clone(),
document_id,
diff --git a/services/document_text_extractor/src/service/db/create_document_text.rs b/services/document_text_extractor/src/service/db/create_document_text.rs
index a71cc461d1..60fd30e251 100644
--- a/services/document_text_extractor/src/service/db/create_document_text.rs
+++ b/services/document_text_extractor/src/service/db/create_document_text.rs
@@ -1,15 +1,19 @@
+use super::DocumentTextOutcome;
use anyhow::Result;
use lambda_runtime::tracing;
use sqlx::{Pool, Postgres};
+#[cfg(test)]
+mod test;
+
#[tracing::instrument(skip(db))]
pub async fn create_document_text(
db: Pool<Postgres>,
document_id: &str,
text: &str,
token_count: i64,
-) -> Result<()> {
- sqlx::query_as!(
+) -> Result<DocumentTextOutcome> {
+ let inserted = sqlx::query_as!(
DocumentText,
r#"
INSERT INTO "DocumentText" ("documentId", "content", "tokenCount")
@@ -22,7 +26,17 @@ pub async fn create_document_text(
token_count
)
.execute(&db)
- .await?;
+ .await;
- Ok(())
+ match inserted {
+ Ok(_) => Ok(DocumentTextOutcome::Stored),
+ // "DocumentText" has a single foreign key, to "Document"
+ Err(e)
+ if e.as_database_error()
+ .is_some_and(|e| e.is_foreign_key_violation()) =>
+ {
+ Ok(DocumentTextOutcome::DocumentMissing)
+ }
+ Err(e) => Err(e.into()),
+ }
}
diff --git a/services/document_text_extractor/src/service/db/create_document_text/test.rs b/services/document_text_extractor/src/service/db/create_document_text/test.rs
new file mode 100644
index 0000000000..7d9c8340f0
--- /dev/null
+++ b/services/document_text_extractor/src/service/db/create_document_text/test.rs
@@ -0,0 +1,23 @@
+use super::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+
+async fn check(pool: &Pool<Postgres>, document_id: &str, expected: DocumentTextOutcome) {
+ let outcome = create_document_text(pool.clone(), document_id, "extracted text", 2)
+ .await
+ .unwrap();
+ assert_eq!(outcome, expected, "{document_id}");
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../fixtures", scripts("document_text"))
+)]
+async fn text_for_a_deleted_document_is_reported_missing_instead_of_failing(pool: Pool<Postgres>) {
+ check(&pool, "document-one", DocumentTextOutcome::Stored).await;
+ check(
+ &pool,
+ "deleted-document",
+ DocumentTextOutcome::DocumentMissing,
+ )
+ .await;
+}