Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(convert): send docx job updates only for jobs a client tracks (#9566)

teo4491a64115ff
diff --git a/crates/model/src/convert.rs b/crates/model/src/convert.rs
index 167082bfc7..d4ef924501 100644
--- a/crates/model/src/convert.rs
+++ b/crates/model/src/convert.rs
@@ -35,4 +35,15 @@ pub struct ConvertQueueMessage {
/// The target file type. Falls back to the `to_key` extension.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub to_file_type: Option<FileType>,
+ /// Whether a client job is tracking `job_id`, so the convert service
+ /// should report the conversion's outcome to the job update handler.
+ /// Producers that mint their own id must set this to `false`: the handler
+ /// rejects updates for unknown jobs, and Lambda retries each rejection.
+ /// Defaults to `true` for messages from producers that predate the field.
+ #[serde(default = "notify_job_default")]
+ pub notify_job: bool,
+}
+
+fn notify_job_default() -> bool {
+ true
}
diff --git a/services/convert_service/AGENTS.md b/services/convert_service/AGENTS.md
index 546aeda03a..6a055ed901 100644
--- a/services/convert_service/AGENTS.md
+++ b/services/convert_service/AGENTS.md
@@ -48,6 +48,11 @@ synchronously via HTTP or asynchronously via SQS.
- The convert queue's visibility timeout (`infra/stacks/convert-service`)
must stay above `conversion_timeout`; failed messages go to the DLQ after
their second receive.
+- DOCX conversions report `Completed`/`Failed` to the job update Lambda
+ (`WEB_SOCKET_RESPONSE_LAMBDA`) only when the message's `notify_job` is set
+ (`job_update` in `src/process/convert.rs`). The Lambda fails on job ids it
+ doesn't track, and async invocation retries each failure twice, so a
+ producer that mints its own `job_id` must send `notify_job: false`.
## Related
diff --git a/services/convert_service/src/api/backfill/backfill_docx.rs b/services/convert_service/src/api/backfill/backfill_docx.rs
index facdd0c804..2b651ef278 100644
--- a/services/convert_service/src/api/backfill/backfill_docx.rs
+++ b/services/convert_service/src/api/backfill/backfill_docx.rs
@@ -184,6 +184,7 @@ async fn process_docx(
to_key: to_key.clone(),
from_file_type: None,
to_file_type: None,
+ notify_job: false,
}));
}
diff --git a/services/convert_service/src/process/convert.rs b/services/convert_service/src/process/convert.rs
index e2c20903f4..a90c2c93b0 100644
--- a/services/convert_service/src/process/convert.rs
+++ b/services/convert_service/src/process/convert.rs
@@ -71,58 +71,15 @@ pub async fn process_message(
tracing::error!(error=?e, job_id=%job_id, "unable to cleanup folder");
}
- match result {
- Ok(_) => {
- if is_docx_key(&req.from_key) {
- tracing::trace!("sending success message to lambda");
- lambda_client
- .invoke_event(
- &WEB_SOCKET_RESPONSE_LAMBDA,
- &DocxUploadJobResult {
- job_id: Cow::Borrowed(&job_id),
- status: "Completed".into(),
- job_type: "docx_upload".into(), // TODO: will need to pass in job type to the convert message eventually
- data: DocxUploadJobData {
- error: false,
- message: None,
- data: Some(DocxUploadJobSuccessDataInner {
- converted: true,
- bom_parts: vec![BomPart {
- sha: "bogus".to_string(),
- path: "bogus".to_string(),
- id: "bogus".to_string(),
- document_bom_id: 1,
- }],
- }),
- },
- },
- )
- .await?;
- }
- }
- Err(e) => {
- if is_docx_key(&req.from_key) {
- tracing::trace!("sending failure message to lambda");
- lambda_client
- .invoke_event(
- &WEB_SOCKET_RESPONSE_LAMBDA,
- &DocxUploadJobResult {
- job_id: Cow::Borrowed(&job_id),
- status: "Failed".into(),
- job_type: "docx_upload".into(), // TODO: will need to pass in job type to the convert message eventually
- data: DocxUploadJobData {
- error: true,
- message: Some("error converting document".to_string()),
- data: None,
- },
- },
- )
- .await?;
- }
- return Err(e);
- }
+ if let Some(update) = job_update(&job_id, &req, result.is_ok()) {
+ tracing::trace!(status=%update.status, "sending job update to lambda");
+ lambda_client
+ .invoke_event(&WEB_SOCKET_RESPONSE_LAMBDA, &update)
+ .await?;
}
+ result?;
+
if let Err(e) = worker.cleanup_message(message).await {
tracing::error!(error=?e, job_id=&job_id, "failed to cleanup message");
}
@@ -130,6 +87,53 @@ pub async fn process_message(
Ok(())
}
+/// The job update to send for a finished conversion, if a client job is
+/// waiting on it.
+pub(super) fn job_update<'a>(
+ job_id: &'a str,
+ req: &ConvertQueueMessage,
+ converted: bool,
+) -> Option<DocxUploadJobResult<'a>> {
+ if !req.notify_job || !is_docx_key(&req.from_key) {
+ return None;
+ }
+
+ // TODO: will need to pass in job type to the convert message eventually
+ let job_type = "docx_upload".into();
+
+ if !converted {
+ return Some(DocxUploadJobResult {
+ job_id: Cow::Borrowed(job_id),
+ status: "Failed".into(),
+ job_type,
+ data: DocxUploadJobData {
+ error: true,
+ message: Some("error converting document".to_string()),
+ data: None,
+ },
+ });
+ }
+
+ Some(DocxUploadJobResult {
+ job_id: Cow::Borrowed(job_id),
+ status: "Completed".into(),
+ job_type,
+ data: DocxUploadJobData {
+ error: false,
+ message: None,
+ data: Some(DocxUploadJobSuccessDataInner {
+ converted: true,
+ bom_parts: vec![BomPart {
+ sha: "bogus".to_string(),
+ path: "bogus".to_string(),
+ id: "bogus".to_string(),
+ document_bom_id: 1,
+ }],
+ }),
+ },
+ })
+}
+
pub async fn convert(
job_id: &str,
req: &ConvertQueueMessage,
diff --git a/services/convert_service/src/process/test.rs b/services/convert_service/src/process/test.rs
index 8fb73affa5..1e48ae62b5 100644
--- a/services/convert_service/src/process/test.rs
+++ b/services/convert_service/src/process/test.rs
@@ -436,3 +436,99 @@ mod job_cleanup {
check(|job| std::fs::write(job, b"not a folder").unwrap(), false);
}
}
+
+mod job_updates {
+ //! What a finished conversion reports to the job update handler, which
+ //! rejects updates for jobs it does not track.
+
+ use model::convert::ConvertQueueMessage;
+ use serde_json::json;
+
+ use crate::process::convert::job_update;
+
+ #[test]
+ fn a_tracked_docx_conversion_reports_completion() {
+ let req: ConvertQueueMessage = serde_json::from_value(json!({
+ "job_id": "job-1",
+ "from_bucket": "docx-upload",
+ "to_bucket": "doc-storage",
+ "from_key": "owner/doc/1.docx",
+ "to_key": "owner/doc/converted.pdf",
+ "notify_job": true,
+ }))
+ .unwrap();
+
+ let update = serde_json::to_value(job_update("job-1", &req, true)).unwrap();
+
+ assert_eq!(update["jobId"], "job-1");
+ assert_eq!(update["status"], "Completed");
+ assert_eq!(update["jobType"], "docx_upload");
+ assert_eq!(update["data"]["error"], false);
+ assert_eq!(update["data"]["data"]["converted"], true);
+ }
+
+ #[test]
+ fn a_tracked_docx_conversion_reports_failure() {
+ let req: ConvertQueueMessage = serde_json::from_value(json!({
+ "job_id": "job-1",
+ "from_bucket": "docx-upload",
+ "to_bucket": "doc-storage",
+ "from_key": "owner/doc/1.docx",
+ "to_key": "owner/doc/converted.pdf",
+ "notify_job": true,
+ }))
+ .unwrap();
+
+ let update = serde_json::to_value(job_update("job-1", &req, false)).unwrap();
+
+ assert_eq!(update["status"], "Failed");
+ assert_eq!(update["data"]["error"], true);
+ assert_eq!(update["data"]["message"], "error converting document");
+ }
+
+ #[test]
+ fn a_conversion_without_a_job_reports_nothing() {
+ let req: ConvertQueueMessage = serde_json::from_value(json!({
+ "job_id": "minted",
+ "from_bucket": "docx-upload",
+ "to_bucket": "doc-storage",
+ "from_key": "owner/doc/1.docx",
+ "to_key": "owner/doc/converted.pdf",
+ "notify_job": false,
+ }))
+ .unwrap();
+
+ assert!(job_update("minted", &req, true).is_none());
+ assert!(job_update("minted", &req, false).is_none());
+ }
+
+ #[test]
+ fn messages_from_older_producers_still_report() {
+ let req: ConvertQueueMessage = serde_json::from_value(json!({
+ "job_id": "job-1",
+ "from_bucket": "docx-upload",
+ "to_bucket": "doc-storage",
+ "from_key": "owner/doc/1.docx",
+ "to_key": "owner/doc/converted.pdf",
+ }))
+ .unwrap();
+
+ assert!(job_update("job-1", &req, false).is_some());
+ }
+
+ #[test]
+ fn non_docx_conversions_report_nothing() {
+ let req: ConvertQueueMessage = serde_json::from_value(json!({
+ "job_id": "job-1",
+ "from_bucket": "doc-storage",
+ "to_bucket": "doc-storage",
+ "from_key": "owner/doc/13664129",
+ "to_key": "owner/doc/upgraded.docx",
+ "from_file_type": "doc",
+ "to_file_type": "docx",
+ }))
+ .unwrap();
+
+ assert!(job_update("job-1", &req, true).is_none());
+ }
+}
diff --git a/services/document_upload_finalizer_handler/src/outbound.rs b/services/document_upload_finalizer_handler/src/outbound.rs
index 6387d7c6cc..0a61d5b8f8 100644
--- a/services/document_upload_finalizer_handler/src/outbound.rs
+++ b/services/document_upload_finalizer_handler/src/outbound.rs
@@ -206,6 +206,7 @@ pub fn upgrade_convert_message(
// Versioned keys have no extension, so the types travel explicitly.
from_file_type: Some(request.from),
to_file_type: Some(request.to),
+ notify_job: false,
})
}
diff --git a/services/docx_unzip_handler/src/service/document/process_document.rs b/services/docx_unzip_handler/src/service/document/process_document.rs
index d4fdd58c2c..b2969e9e36 100644
--- a/services/docx_unzip_handler/src/service/document/process_document.rs
+++ b/services/docx_unzip_handler/src/service/document/process_document.rs
@@ -37,31 +37,20 @@ pub async fn process(
)
.await?;
- let job_id = if let Some((job_id, _)) = job {
- job_id
- } else {
- tracing::warn!("no job id found. making a new uuid");
- macro_uuid::generate_uuid_v7().to_string()
- };
+ if job.is_none() {
+ tracing::info!("no upload job for document, converting without job updates");
+ }
// send convert message
tracing::trace!("queueing convert");
if let Err(e) = ctx
.sqs_client
- .enqueue_convert_queue_message(ConvertQueueMessage {
- job_id,
- from_bucket: bucket.clone(),
- from_key: document_key_parts.to_key(),
- // The key for storing a converted version of a file is
- // "{owner}/{document_id}/converted.{file_extension}"
- to_key: build_docx_to_pdf_converted_document_key(
- &document_key_parts.owner,
- &document_key_parts.document_id,
- ),
- to_bucket: ctx.config.document_storage_bucket.clone(),
- from_file_type: None,
- to_file_type: None,
- })
+ .enqueue_convert_queue_message(convert_message(
+ job.map(|(job_id, _)| job_id),
+ bucket,
+ &document_key_parts,
+ &ctx.config.document_storage_bucket,
+ ))
.await
{
tracing::error!(error=?e, "unable to send convert message");
@@ -150,3 +139,35 @@ pub async fn process(
Ok(())
}
+
+/// The message converting an uploaded DOCX to PDF. Only a conversion with an
+/// upload job reports its outcome; one without a job gets a fresh id for
+/// logs and folder names.
+pub(super) fn convert_message(
+ upload_job_id: Option<String>,
+ bucket: &str,
+ document_key_parts: &DocumentKeyParts,
+ document_storage_bucket: &str,
+) -> ConvertQueueMessage {
+ let notify_job = upload_job_id.is_some();
+ let job_id = upload_job_id.unwrap_or_else(|| macro_uuid::generate_uuid_v7().to_string());
+
+ ConvertQueueMessage {
+ job_id,
+ from_bucket: bucket.to_string(),
+ from_key: document_key_parts.to_key(),
+ // The key for storing a converted version of a file is
+ // "{owner}/{document_id}/converted.{file_extension}"
+ to_key: build_docx_to_pdf_converted_document_key(
+ &document_key_parts.owner,
+ &document_key_parts.document_id,
+ ),
+ to_bucket: document_storage_bucket.to_string(),
+ from_file_type: None,
+ to_file_type: None,
+ notify_job,
+ }
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/services/docx_unzip_handler/src/service/document/process_document/tests.rs b/services/docx_unzip_handler/src/service/document/process_document/tests.rs
new file mode 100644
index 0000000000..80051c7042
--- /dev/null
+++ b/services/docx_unzip_handler/src/service/document/process_document/tests.rs
@@ -0,0 +1,33 @@
+use super::convert_message;
+use crate::models::DocumentKeyParts;
+
+#[test]
+fn a_tracked_upload_reports_to_its_job() {
+ let parts = DocumentKeyParts::from_s3_key("macro%7Cuser%40example.com/doc-1/42.docx").unwrap();
+
+ let message = convert_message(
+ Some("upload-job".to_string()),
+ "docx-upload",
+ &parts,
+ "doc-storage",
+ );
+
+ assert_eq!(message.job_id, "upload-job");
+ assert!(message.notify_job);
+ assert_eq!(message.from_bucket, "docx-upload");
+ assert_eq!(message.from_key, "macro|[email protected]/doc-1/42.docx");
+ assert_eq!(message.to_bucket, "doc-storage");
+ assert_eq!(message.to_key, "macro|[email protected]/doc-1/converted.pdf");
+}
+
+#[test]
+fn an_upload_without_a_job_converts_without_job_updates() {
+ let parts = DocumentKeyParts::from_s3_key("macro%7Cuser%40example.com/doc-1/42.docx").unwrap();
+
+ let first = convert_message(None, "docx-upload", &parts, "doc-storage");
+ let second = convert_message(None, "docx-upload", &parts, "doc-storage");
+
+ assert!(!first.notify_job);
+ assert_ne!(first.job_id, second.job_id);
+ assert_eq!(first.to_key, "macro|[email protected]/doc-1/converted.pdf");
+}