Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(convert): error process_message spans by their failure, not cleanup (#9653)
teo914a8268ef59
diff --git a/Cargo.lock b/Cargo.lockindex 6f6322eb81..78ac464cc3 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5276,6 +5276,8 @@ dependencies = ["macro_uuid","model","nix 0.29.0",+ "opentelemetry",+ "opentelemetry_sdk","pptx_engine","remote_env_var","rs-libreoffice-bindings",@@ -5292,6 +5294,8 @@ dependencies = ["tower 0.5.3","tower-http","tracing",+ "tracing-opentelemetry",+ "tracing-subscriber","utoipa","utoipa-swagger-ui","workspace-hack",diff --git a/deny.toml b/deny.tomlindex 424903f4ac..5e9e89faa6 100644--- a/deny.toml+++ b/deny.toml@@ -225,6 +225,7 @@ deny = ["ai_toolset", # GenAI span tests"channels", # Dev logging only"coding_agent_worker", # Standalone daemon logging setup+ "convert_service", # process_message span status tests only (dev-dependency)"cursor_cloud_agents", # Standalone ACP binary logging; stdout is the protocol, so macro_entrypoint cannot own it"dictation", # Metadata tracing and content-redaction tests only"document_storage_service", # GraphQL telemetry span and log tests only (dev-dependency)diff --git a/services/convert_service/AGENTS.md b/services/convert_service/AGENTS.mdindex 6a055ed901..04bae9b567 100644--- a/services/convert_service/AGENTS.md+++ b/services/convert_service/AGENTS.md@@ -45,6 +45,10 @@ synchronously via HTTP or asynchronously via SQS.(`loadComponentFromURL returned an empty reference`, `Failed to savedocument`) next to `No space left on device`; check`ecs.fargate.ephemeral_storage.utilized` before debugging LibreOffice.+- Any `tracing::error!` inside `process_message` errors its span and counts+ toward the internal span error monitor; log expected outcomes (such as a+ job folder that is already gone) below `error`, and let the returned error+ mark real failures.- The convert queue's visibility timeout (`infra/stacks/convert-service`)must stay above `conversion_timeout`; failed messages go to the DLQ aftertheir second receive.diff --git a/services/convert_service/Cargo.toml b/services/convert_service/Cargo.tomlindex 0b2bfe6c3c..3d63fa41b5 100644--- a/services/convert_service/Cargo.toml+++ b/services/convert_service/Cargo.toml@@ -63,5 +63,9 @@ workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }[dev-dependencies]docx_engine = { path = "../../crates/docx_engine" }+opentelemetry = { workspace = true, features = ["trace"] }+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }pptx_engine = { path = "../../crates/pptx_engine" }tempfile = { workspace = true }+tracing-opentelemetry = { workspace = true }+tracing-subscriber = { workspace = true }diff --git a/services/convert_service/src/process/convert.rs b/services/convert_service/src/process/convert.rsindex a90c2c93b0..58b8c15149 100644--- a/services/convert_service/src/process/convert.rs+++ b/services/convert_service/src/process/convert.rs@@ -22,7 +22,12 @@ pub(super) fn is_docx_key(key: &str) -> bool {/// Processes a message from the convert queue/// Returns the job id and the convert queue message-#[tracing::instrument(skip(worker, s3_client, lambda_client, message), fields(message_id = message.message_id))]+///+/// The span is errored by the returned error. Cleaning up a folder that was+/// never created does not error it, so a job that failed before creating its+/// folder still shows that failure; any other cleanup failure logs an error+/// and errors the span.+#[tracing::instrument(err(Debug), skip(worker, s3_client, lambda_client, message), fields(message_id = message.message_id))]pub async fn process_message(worker: &sqs_worker::SQSWorker,s3_client: &s3_client::S3,diff --git a/services/convert_service/src/process/test.rs b/services/convert_service/src/process/test.rsindex 1e48ae62b5..e19efd91ea 100644--- a/services/convert_service/src/process/test.rs+++ b/services/convert_service/src/process/test.rs@@ -437,6 +437,117 @@ mod job_cleanup {}}+mod process_message_span {+ //! Exports `process_message` spans with the error-event-to-status mapping+ //! the service's OTel layer uses, so a span's status here matches what+ //! Datadog counts as a span error.++ use aws_sdk_sqs::types::{Message, MessageAttributeValue};+ use opentelemetry::trace::{Status, TracerProvider as _};+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};+ use tracing_subscriber::layer::SubscriberExt as _;++ use crate::process::convert::process_message;++ fn s3() -> s3_client::S3 {+ let config = aws_sdk_s3::Config::builder()+ .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())+ .region(aws_sdk_s3::config::Region::new("us-east-1"))+ .build();+ s3_client::S3::new(aws_sdk_s3::Client::from_conf(config))+ }++ fn lambda() -> lambda_client::Lambda {+ let config = aws_sdk_lambda::Config::builder()+ .behavior_version(aws_sdk_lambda::config::BehaviorVersion::latest())+ .region(aws_sdk_lambda::config::Region::new("us-east-1"))+ .build();+ lambda_client::Lambda::new(aws_sdk_lambda::Client::from_conf(config))+ }++ fn worker() -> sqs_worker::SQSWorker {+ let config = aws_sdk_sqs::Config::builder()+ .behavior_version(aws_sdk_sqs::config::BehaviorVersion::latest())+ .region(aws_sdk_sqs::config::Region::new("us-east-1"))+ .build();+ sqs_worker::SQSWorker::new(+ aws_sdk_sqs::Client::from_conf(config),+ "https://sqs.us-east-1.amazonaws.com/0/convert".to_string(),+ 1,+ 0,+ )+ }++ /// A queue message for `job_id` converting `from_key` to `to_key`.+ fn message(job_id: &str, from_key: &str, to_key: &str) -> Message {+ let body = serde_json::json!({+ "job_id": job_id,+ "from_bucket": "from-bucket",+ "to_bucket": "to-bucket",+ "from_key": from_key,+ "to_key": to_key,+ });+ Message::builder()+ .message_id("message")+ .message_attributes(+ "job_id",+ MessageAttributeValue::builder()+ .data_type("String")+ .string_value(job_id)+ .build()+ .unwrap(),+ )+ .body(body.to_string())+ .build()+ }++ /// Runs `process_message` and returns its result and exported span.+ async fn run(message: &Message) -> (anyhow::Result<()>, SpanData) {+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer()+ .with_tracer(provider.tracer("test"))+ .with_error_events_to_status(true);+ let result = {+ let _guard =+ tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ tracing::callsite::rebuild_interest_cache();+ process_message(&worker(), &s3(), &lambda(), message).await+ };++ let span = exporter+ .get_finished_spans()+ .unwrap()+ .into_iter()+ .find(|span| span.name == "process_message")+ .expect("process_message span");+ (result, span)+ }++ #[tokio::test]+ async fn a_job_that_fails_before_creating_its_folder_errors_the_span_by_its_failure() {+ let job_id = "process-message-span-unsupported";+ let (result, span) = run(&message(job_id, "uploads/notes.txt", "uploads/notes.pdf")).await;++ assert!(result.is_err());+ assert!(!std::path::Path::new(job_id).exists());+ let Status::Error { description } = &span.status else {+ panic!("expected an errored span, got {:?}", span.status);+ };+ assert!(+ description.contains("unsupported conversion"),+ "span error should be the conversion failure: {description}"+ );+ let events: Vec<_> = span.events.iter().map(|event| &*event.name).collect();+ assert!(+ !events.contains(&"unable to cleanup folder"),+ "cleaning up a missing folder must not log an error: {events:?}"+ );+ }+}+mod job_updates {//! What a finished conversion reports to the job update handler, which//! rejects updates for jobs it does not track.