Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(convert): error process_message spans by their failure, not cleanup (#9653)

teo914a8268ef59
diff --git a/Cargo.lock b/Cargo.lock
index 6f6322eb81..78ac464cc3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5276,6 +5276,8 @@ dependencies = [
"macro_uuid",
"model",
"nix 0.29.0",
+ "opentelemetry",
+ "opentelemetry_sdk",
"pptx_engine",
"remote_env_var",
"rs-libreoffice-bindings",
@@ -5292,6 +5294,8 @@ dependencies = [
"tower 0.5.3",
"tower-http",
"tracing",
+ "tracing-opentelemetry",
+ "tracing-subscriber",
"utoipa",
"utoipa-swagger-ui",
"workspace-hack",
diff --git a/deny.toml b/deny.toml
index 424903f4ac..5e9e89faa6 100644
--- a/deny.toml
+++ b/deny.toml
@@ -225,6 +225,7 @@ deny = [
"ai_toolset", # GenAI span tests
"channels", # Dev logging only
"coding_agent_worker", # Standalone daemon logging setup
+ "convert_service", # process_message span status tests only (dev-dependency)
"cursor_cloud_agents", # Standalone ACP binary logging; stdout is the protocol, so macro_entrypoint cannot own it
"dictation", # Metadata tracing and content-redaction tests only
"document_storage_service", # GraphQL telemetry span and log tests only (dev-dependency)
diff --git a/services/convert_service/AGENTS.md b/services/convert_service/AGENTS.md
index 6a055ed901..04bae9b567 100644
--- a/services/convert_service/AGENTS.md
+++ b/services/convert_service/AGENTS.md
@@ -45,6 +45,10 @@ synchronously via HTTP or asynchronously via SQS.
(`loadComponentFromURL returned an empty reference`, `Failed to save
document`) next to `No space left on device`; check
`ecs.fargate.ephemeral_storage.utilized` before debugging LibreOffice.
+- Any `tracing::error!` inside `process_message` errors its span and counts
+ toward the internal span error monitor; log expected outcomes (such as a
+ job folder that is already gone) below `error`, and let the returned error
+ mark real failures.
- The convert queue's visibility timeout (`infra/stacks/convert-service`)
must stay above `conversion_timeout`; failed messages go to the DLQ after
their second receive.
diff --git a/services/convert_service/Cargo.toml b/services/convert_service/Cargo.toml
index 0b2bfe6c3c..3d63fa41b5 100644
--- a/services/convert_service/Cargo.toml
+++ b/services/convert_service/Cargo.toml
@@ -63,5 +63,9 @@ workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }
[dev-dependencies]
docx_engine = { path = "../../crates/docx_engine" }
+opentelemetry = { workspace = true, features = ["trace"] }
+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }
pptx_engine = { path = "../../crates/pptx_engine" }
tempfile = { workspace = true }
+tracing-opentelemetry = { workspace = true }
+tracing-subscriber = { workspace = true }
diff --git a/services/convert_service/src/process/convert.rs b/services/convert_service/src/process/convert.rs
index a90c2c93b0..58b8c15149 100644
--- a/services/convert_service/src/process/convert.rs
+++ b/services/convert_service/src/process/convert.rs
@@ -22,7 +22,12 @@ pub(super) fn is_docx_key(key: &str) -> bool {
/// Processes a message from the convert queue
/// Returns the job id and the convert queue message
-#[tracing::instrument(skip(worker, s3_client, lambda_client, message), fields(message_id = message.message_id))]
+///
+/// The span is errored by the returned error. Cleaning up a folder that was
+/// never created does not error it, so a job that failed before creating its
+/// folder still shows that failure; any other cleanup failure logs an error
+/// and errors the span.
+#[tracing::instrument(err(Debug), skip(worker, s3_client, lambda_client, message), fields(message_id = message.message_id))]
pub async fn process_message(
worker: &sqs_worker::SQSWorker,
s3_client: &s3_client::S3,
diff --git a/services/convert_service/src/process/test.rs b/services/convert_service/src/process/test.rs
index 1e48ae62b5..e19efd91ea 100644
--- a/services/convert_service/src/process/test.rs
+++ b/services/convert_service/src/process/test.rs
@@ -437,6 +437,117 @@ mod job_cleanup {
}
}
+mod process_message_span {
+ //! Exports `process_message` spans with the error-event-to-status mapping
+ //! the service's OTel layer uses, so a span's status here matches what
+ //! Datadog counts as a span error.
+
+ use aws_sdk_sqs::types::{Message, MessageAttributeValue};
+ use opentelemetry::trace::{Status, TracerProvider as _};
+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
+ use tracing_subscriber::layer::SubscriberExt as _;
+
+ use crate::process::convert::process_message;
+
+ fn s3() -> s3_client::S3 {
+ let config = aws_sdk_s3::Config::builder()
+ .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
+ .region(aws_sdk_s3::config::Region::new("us-east-1"))
+ .build();
+ s3_client::S3::new(aws_sdk_s3::Client::from_conf(config))
+ }
+
+ fn lambda() -> lambda_client::Lambda {
+ let config = aws_sdk_lambda::Config::builder()
+ .behavior_version(aws_sdk_lambda::config::BehaviorVersion::latest())
+ .region(aws_sdk_lambda::config::Region::new("us-east-1"))
+ .build();
+ lambda_client::Lambda::new(aws_sdk_lambda::Client::from_conf(config))
+ }
+
+ fn worker() -> sqs_worker::SQSWorker {
+ let config = aws_sdk_sqs::Config::builder()
+ .behavior_version(aws_sdk_sqs::config::BehaviorVersion::latest())
+ .region(aws_sdk_sqs::config::Region::new("us-east-1"))
+ .build();
+ sqs_worker::SQSWorker::new(
+ aws_sdk_sqs::Client::from_conf(config),
+ "https://sqs.us-east-1.amazonaws.com/0/convert".to_string(),
+ 1,
+ 0,
+ )
+ }
+
+ /// A queue message for `job_id` converting `from_key` to `to_key`.
+ fn message(job_id: &str, from_key: &str, to_key: &str) -> Message {
+ let body = serde_json::json!({
+ "job_id": job_id,
+ "from_bucket": "from-bucket",
+ "to_bucket": "to-bucket",
+ "from_key": from_key,
+ "to_key": to_key,
+ });
+ Message::builder()
+ .message_id("message")
+ .message_attributes(
+ "job_id",
+ MessageAttributeValue::builder()
+ .data_type("String")
+ .string_value(job_id)
+ .build()
+ .unwrap(),
+ )
+ .body(body.to_string())
+ .build()
+ }
+
+ /// Runs `process_message` and returns its result and exported span.
+ async fn run(message: &Message) -> (anyhow::Result<()>, SpanData) {
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer()
+ .with_tracer(provider.tracer("test"))
+ .with_error_events_to_status(true);
+ let result = {
+ let _guard =
+ tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ tracing::callsite::rebuild_interest_cache();
+ process_message(&worker(), &s3(), &lambda(), message).await
+ };
+
+ let span = exporter
+ .get_finished_spans()
+ .unwrap()
+ .into_iter()
+ .find(|span| span.name == "process_message")
+ .expect("process_message span");
+ (result, span)
+ }
+
+ #[tokio::test]
+ async fn a_job_that_fails_before_creating_its_folder_errors_the_span_by_its_failure() {
+ let job_id = "process-message-span-unsupported";
+ let (result, span) = run(&message(job_id, "uploads/notes.txt", "uploads/notes.pdf")).await;
+
+ assert!(result.is_err());
+ assert!(!std::path::Path::new(job_id).exists());
+ let Status::Error { description } = &span.status else {
+ panic!("expected an errored span, got {:?}", span.status);
+ };
+ assert!(
+ description.contains("unsupported conversion"),
+ "span error should be the conversion failure: {description}"
+ );
+ let events: Vec<_> = span.events.iter().map(|event| &*event.name).collect();
+ assert!(
+ !events.contains(&"unable to cleanup folder"),
+ "cleaning up a missing folder must not log an error: {events:?}"
+ );
+ }
+}
+
mod job_updates {
//! What a finished conversion reports to the job update handler, which
//! rejects updates for jobs it does not track.