Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(convert): send docx job updates only for jobs a client tracks (#9566)
teo4491a64115ff
diff --git a/crates/model/src/convert.rs b/crates/model/src/convert.rsindex 167082bfc7..d4ef924501 100644--- a/crates/model/src/convert.rs+++ b/crates/model/src/convert.rs@@ -35,4 +35,15 @@ pub struct ConvertQueueMessage {/// The target file type. Falls back to the `to_key` extension.#[serde(default, skip_serializing_if = "Option::is_none")]pub to_file_type: Option<FileType>,+ /// Whether a client job is tracking `job_id`, so the convert service+ /// should report the conversion's outcome to the job update handler.+ /// Producers that mint their own id must set this to `false`: the handler+ /// rejects updates for unknown jobs, and Lambda retries each rejection.+ /// Defaults to `true` for messages from producers that predate the field.+ #[serde(default = "notify_job_default")]+ pub notify_job: bool,+}++fn notify_job_default() -> bool {+ true}diff --git a/services/convert_service/AGENTS.md b/services/convert_service/AGENTS.mdindex 546aeda03a..6a055ed901 100644--- a/services/convert_service/AGENTS.md+++ b/services/convert_service/AGENTS.md@@ -48,6 +48,11 @@ synchronously via HTTP or asynchronously via SQS.- The convert queue's visibility timeout (`infra/stacks/convert-service`)must stay above `conversion_timeout`; failed messages go to the DLQ aftertheir second receive.+- DOCX conversions report `Completed`/`Failed` to the job update Lambda+ (`WEB_SOCKET_RESPONSE_LAMBDA`) only when the message's `notify_job` is set+ (`job_update` in `src/process/convert.rs`). The Lambda fails on job ids it+ doesn't track, and async invocation retries each failure twice, so a+ producer that mints its own `job_id` must send `notify_job: false`.## Relateddiff --git a/services/convert_service/src/api/backfill/backfill_docx.rs b/services/convert_service/src/api/backfill/backfill_docx.rsindex facdd0c804..2b651ef278 100644--- a/services/convert_service/src/api/backfill/backfill_docx.rs+++ b/services/convert_service/src/api/backfill/backfill_docx.rs@@ -184,6 +184,7 @@ async fn process_docx(to_key: to_key.clone(),from_file_type: None,to_file_type: None,+ notify_job: false,}));}diff --git a/services/convert_service/src/process/convert.rs b/services/convert_service/src/process/convert.rsindex e2c20903f4..a90c2c93b0 100644--- a/services/convert_service/src/process/convert.rs+++ b/services/convert_service/src/process/convert.rs@@ -71,58 +71,15 @@ pub async fn process_message(tracing::error!(error=?e, job_id=%job_id, "unable to cleanup folder");}- match result {- Ok(_) => {- if is_docx_key(&req.from_key) {- tracing::trace!("sending success message to lambda");- lambda_client- .invoke_event(- &WEB_SOCKET_RESPONSE_LAMBDA,- &DocxUploadJobResult {- job_id: Cow::Borrowed(&job_id),- status: "Completed".into(),- job_type: "docx_upload".into(), // TODO: will need to pass in job type to the convert message eventually- data: DocxUploadJobData {- error: false,- message: None,- data: Some(DocxUploadJobSuccessDataInner {- converted: true,- bom_parts: vec![BomPart {- sha: "bogus".to_string(),- path: "bogus".to_string(),- id: "bogus".to_string(),- document_bom_id: 1,- }],- }),- },- },- )- .await?;- }- }- Err(e) => {- if is_docx_key(&req.from_key) {- tracing::trace!("sending failure message to lambda");- lambda_client- .invoke_event(- &WEB_SOCKET_RESPONSE_LAMBDA,- &DocxUploadJobResult {- job_id: Cow::Borrowed(&job_id),- status: "Failed".into(),- job_type: "docx_upload".into(), // TODO: will need to pass in job type to the convert message eventually- data: DocxUploadJobData {- error: true,- message: Some("error converting document".to_string()),- data: None,- },- },- )- .await?;- }- return Err(e);- }+ if let Some(update) = job_update(&job_id, &req, result.is_ok()) {+ tracing::trace!(status=%update.status, "sending job update to lambda");+ lambda_client+ .invoke_event(&WEB_SOCKET_RESPONSE_LAMBDA, &update)+ .await?;}+ result?;+if let Err(e) = worker.cleanup_message(message).await {tracing::error!(error=?e, job_id=&job_id, "failed to cleanup message");}@@ -130,6 +87,53 @@ pub async fn process_message(Ok(())}+/// The job update to send for a finished conversion, if a client job is+/// waiting on it.+pub(super) fn job_update<'a>(+ job_id: &'a str,+ req: &ConvertQueueMessage,+ converted: bool,+) -> Option<DocxUploadJobResult<'a>> {+ if !req.notify_job || !is_docx_key(&req.from_key) {+ return None;+ }++ // TODO: will need to pass in job type to the convert message eventually+ let job_type = "docx_upload".into();++ if !converted {+ return Some(DocxUploadJobResult {+ job_id: Cow::Borrowed(job_id),+ status: "Failed".into(),+ job_type,+ data: DocxUploadJobData {+ error: true,+ message: Some("error converting document".to_string()),+ data: None,+ },+ });+ }++ Some(DocxUploadJobResult {+ job_id: Cow::Borrowed(job_id),+ status: "Completed".into(),+ job_type,+ data: DocxUploadJobData {+ error: false,+ message: None,+ data: Some(DocxUploadJobSuccessDataInner {+ converted: true,+ bom_parts: vec![BomPart {+ sha: "bogus".to_string(),+ path: "bogus".to_string(),+ id: "bogus".to_string(),+ document_bom_id: 1,+ }],+ }),+ },+ })+}+pub async fn convert(job_id: &str,req: &ConvertQueueMessage,diff --git a/services/convert_service/src/process/test.rs b/services/convert_service/src/process/test.rsindex 8fb73affa5..1e48ae62b5 100644--- a/services/convert_service/src/process/test.rs+++ b/services/convert_service/src/process/test.rs@@ -436,3 +436,99 @@ mod job_cleanup {check(|job| std::fs::write(job, b"not a folder").unwrap(), false);}}++mod job_updates {+ //! What a finished conversion reports to the job update handler, which+ //! rejects updates for jobs it does not track.++ use model::convert::ConvertQueueMessage;+ use serde_json::json;++ use crate::process::convert::job_update;++ #[test]+ fn a_tracked_docx_conversion_reports_completion() {+ let req: ConvertQueueMessage = serde_json::from_value(json!({+ "job_id": "job-1",+ "from_bucket": "docx-upload",+ "to_bucket": "doc-storage",+ "from_key": "owner/doc/1.docx",+ "to_key": "owner/doc/converted.pdf",+ "notify_job": true,+ }))+ .unwrap();++ let update = serde_json::to_value(job_update("job-1", &req, true)).unwrap();++ assert_eq!(update["jobId"], "job-1");+ assert_eq!(update["status"], "Completed");+ assert_eq!(update["jobType"], "docx_upload");+ assert_eq!(update["data"]["error"], false);+ assert_eq!(update["data"]["data"]["converted"], true);+ }++ #[test]+ fn a_tracked_docx_conversion_reports_failure() {+ let req: ConvertQueueMessage = serde_json::from_value(json!({+ "job_id": "job-1",+ "from_bucket": "docx-upload",+ "to_bucket": "doc-storage",+ "from_key": "owner/doc/1.docx",+ "to_key": "owner/doc/converted.pdf",+ "notify_job": true,+ }))+ .unwrap();++ let update = serde_json::to_value(job_update("job-1", &req, false)).unwrap();++ assert_eq!(update["status"], "Failed");+ assert_eq!(update["data"]["error"], true);+ assert_eq!(update["data"]["message"], "error converting document");+ }++ #[test]+ fn a_conversion_without_a_job_reports_nothing() {+ let req: ConvertQueueMessage = serde_json::from_value(json!({+ "job_id": "minted",+ "from_bucket": "docx-upload",+ "to_bucket": "doc-storage",+ "from_key": "owner/doc/1.docx",+ "to_key": "owner/doc/converted.pdf",+ "notify_job": false,+ }))+ .unwrap();++ assert!(job_update("minted", &req, true).is_none());+ assert!(job_update("minted", &req, false).is_none());+ }++ #[test]+ fn messages_from_older_producers_still_report() {+ let req: ConvertQueueMessage = serde_json::from_value(json!({+ "job_id": "job-1",+ "from_bucket": "docx-upload",+ "to_bucket": "doc-storage",+ "from_key": "owner/doc/1.docx",+ "to_key": "owner/doc/converted.pdf",+ }))+ .unwrap();++ assert!(job_update("job-1", &req, false).is_some());+ }++ #[test]+ fn non_docx_conversions_report_nothing() {+ let req: ConvertQueueMessage = serde_json::from_value(json!({+ "job_id": "job-1",+ "from_bucket": "doc-storage",+ "to_bucket": "doc-storage",+ "from_key": "owner/doc/13664129",+ "to_key": "owner/doc/upgraded.docx",+ "from_file_type": "doc",+ "to_file_type": "docx",+ }))+ .unwrap();++ assert!(job_update("job-1", &req, true).is_none());+ }+}diff --git a/services/document_upload_finalizer_handler/src/outbound.rs b/services/document_upload_finalizer_handler/src/outbound.rsindex 6387d7c6cc..0a61d5b8f8 100644--- a/services/document_upload_finalizer_handler/src/outbound.rs+++ b/services/document_upload_finalizer_handler/src/outbound.rs@@ -206,6 +206,7 @@ pub fn upgrade_convert_message(// Versioned keys have no extension, so the types travel explicitly.from_file_type: Some(request.from),to_file_type: Some(request.to),+ notify_job: false,})}diff --git a/services/docx_unzip_handler/src/service/document/process_document.rs b/services/docx_unzip_handler/src/service/document/process_document.rsindex d4fdd58c2c..b2969e9e36 100644--- a/services/docx_unzip_handler/src/service/document/process_document.rs+++ b/services/docx_unzip_handler/src/service/document/process_document.rs@@ -37,31 +37,20 @@ pub async fn process().await?;- let job_id = if let Some((job_id, _)) = job {- job_id- } else {- tracing::warn!("no job id found. making a new uuid");- macro_uuid::generate_uuid_v7().to_string()- };+ if job.is_none() {+ tracing::info!("no upload job for document, converting without job updates");+ }// send convert messagetracing::trace!("queueing convert");if let Err(e) = ctx.sqs_client- .enqueue_convert_queue_message(ConvertQueueMessage {- job_id,- from_bucket: bucket.clone(),- from_key: document_key_parts.to_key(),- // The key for storing a converted version of a file is- // "{owner}/{document_id}/converted.{file_extension}"- to_key: build_docx_to_pdf_converted_document_key(- &document_key_parts.owner,- &document_key_parts.document_id,- ),- to_bucket: ctx.config.document_storage_bucket.clone(),- from_file_type: None,- to_file_type: None,- })+ .enqueue_convert_queue_message(convert_message(+ job.map(|(job_id, _)| job_id),+ bucket,+ &document_key_parts,+ &ctx.config.document_storage_bucket,+ )).await{tracing::error!(error=?e, "unable to send convert message");@@ -150,3 +139,35 @@ pub async fn process(Ok(())}++/// The message converting an uploaded DOCX to PDF. Only a conversion with an+/// upload job reports its outcome; one without a job gets a fresh id for+/// logs and folder names.+pub(super) fn convert_message(+ upload_job_id: Option<String>,+ bucket: &str,+ document_key_parts: &DocumentKeyParts,+ document_storage_bucket: &str,+) -> ConvertQueueMessage {+ let notify_job = upload_job_id.is_some();+ let job_id = upload_job_id.unwrap_or_else(|| macro_uuid::generate_uuid_v7().to_string());++ ConvertQueueMessage {+ job_id,+ from_bucket: bucket.to_string(),+ from_key: document_key_parts.to_key(),+ // The key for storing a converted version of a file is+ // "{owner}/{document_id}/converted.{file_extension}"+ to_key: build_docx_to_pdf_converted_document_key(+ &document_key_parts.owner,+ &document_key_parts.document_id,+ ),+ to_bucket: document_storage_bucket.to_string(),+ from_file_type: None,+ to_file_type: None,+ notify_job,+ }+}++#[cfg(test)]+mod tests;diff --git a/services/docx_unzip_handler/src/service/document/process_document/tests.rs b/services/docx_unzip_handler/src/service/document/process_document/tests.rsnew file mode 100644index 0000000000..80051c7042--- /dev/null+++ b/services/docx_unzip_handler/src/service/document/process_document/tests.rs@@ -0,0 +1,33 @@+use super::convert_message;+use crate::models::DocumentKeyParts;++#[test]+fn a_tracked_upload_reports_to_its_job() {+ let parts = DocumentKeyParts::from_s3_key("macro%7Cuser%40example.com/doc-1/42.docx").unwrap();++ let message = convert_message(+ Some("upload-job".to_string()),+ "docx-upload",+ &parts,+ "doc-storage",+ );++ assert_eq!(message.job_id, "upload-job");+ assert!(message.notify_job);+ assert_eq!(message.from_bucket, "docx-upload");+ assert_eq!(message.from_key, "macro|[email protected]/doc-1/42.docx");+ assert_eq!(message.to_bucket, "doc-storage");+ assert_eq!(message.to_key, "macro|[email protected]/doc-1/converted.pdf");+}++#[test]+fn an_upload_without_a_job_converts_without_job_updates() {+ let parts = DocumentKeyParts::from_s3_key("macro%7Cuser%40example.com/doc-1/42.docx").unwrap();++ let first = convert_message(None, "docx-upload", &parts, "doc-storage");+ let second = convert_message(None, "docx-upload", &parts, "doc-storage");++ assert!(!first.notify_job);+ assert_ne!(first.job_id, second.job_id);+ assert_eq!(first.to_key, "macro|[email protected]/doc-1/converted.pdf");+}