Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/feat/improve otel (#5136)
Sean Ayec1b6ae29a60b
diff --git a/Cargo.lock b/Cargo.lockindex 25af734a1e..4aec9f4fb7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -8384,6 +8384,8 @@ dependencies = ["opentelemetry","opentelemetry-otlp","opentelemetry_sdk",+ "rootcause",+ "rootcause-tracing","serde_json","tracing","tracing-opentelemetry",@@ -12074,6 +12076,17 @@ dependencies = ["triomphe",]+[[package]]+name = "rootcause-tracing"+version = "0.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f3128c7019d8e7814a832aaa712fe55314d1fa0f0ec53e593bc1b4410962db94"+dependencies = [+ "rootcause",+ "tracing",+ "tracing-subscriber",+]+[[package]]name = "rowan"version = "0.16.1"diff --git a/Cargo.toml b/Cargo.tomlindex 7fd9c994fe..e1e8859d4a 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -202,6 +202,7 @@ reqwest = { version = "0.13", default-features = false, features = [] }rmcp = "1.6.0"rootcause = { version = "0.12" }+rootcause-tracing = "0.12.1"schemars = { version = "1.0.4", features = ["chrono04", "uuid1"] }semver = { version = "1", features = ["serde"] }serde = { version = "1.0.214", features = ["derive"] }diff --git a/crates/macro_entrypoint/Cargo.toml b/crates/macro_entrypoint/Cargo.tomlindex 6ef8f1aeb2..3639cb39f3 100644--- a/crates/macro_entrypoint/Cargo.toml+++ b/crates/macro_entrypoint/Cargo.toml@@ -10,6 +10,8 @@ macro_env = { path = "../macro_env" }opentelemetry = { workspace = true }opentelemetry_sdk = { workspace = true }opentelemetry-otlp = { workspace = true }+rootcause = { workspace = true }+rootcause-tracing = { workspace = true }serde_json = { workspace = true }tracing = { workspace = true }tracing-opentelemetry = { workspace = true }diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rsindex 181bbfd504..9329d5a2a9 100644--- a/crates/macro_entrypoint/src/lib.rs+++ b/crates/macro_entrypoint/src/lib.rs@@ -12,8 +12,13 @@ use macro_env_var::{env_vars, maybe_env_vars};use opentelemetry::trace::TracerProvider as _;use opentelemetry_otlp::WithExportConfig;use opentelemetry_sdk::trace::SdkTracerProvider;+use rootcause::hooks::Hooks;+use rootcause_tracing::{RootcauseLayer, SpanCollector};use tracing_subscriber::{- EnvFilter, Layer, Registry, filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt,+ EnvFilter, Layer, Registry,+ filter::{FilterExt, LevelFilter},+ layer::SubscriberExt,+ util::SubscriberInitExt,};use tracing_tree::HierarchicalLayer;@@ -56,6 +61,13 @@ fn otel_trace_filter(value: Option<&str>) -> EnvFilter {.parse_lossy(value.unwrap_or(""))}+fn install_rootcause_hooks() {+ Hooks::new()+ .report_creation_hook(SpanCollector::new())+ .install()+ .expect("failed to install rootcause tracing hooks");+}+/// unit struct which defines the behaviour for instantiation#[derive(Debug)]pub struct MacroEntrypoint {@@ -106,16 +118,24 @@ impl MacroEntrypoint {pub fn init(self) -> InitializedEntrypoint {let _ = dotenvy::dotenv();std::panic::set_hook(Box::new(tracing_panic::panic_hook));+ install_rootcause_hooks();match (self.env, self.local) {(Environment::Local, LocalOptions { tree_tracing: None }) => {- tracing_subscriber::fmt()+ // Keep rootcause from enabling callsites that RUST_LOG previously disabled.+ let rust_log_filter = rust_log_env_filter();+ let fmt_layer = tracing_subscriber::fmt::layer().with_ansi(true)- .with_env_filter(rust_log_env_filter()).with_file(true).with_line_number(true).pretty()+ .with_filter(rust_log_filter.clone());++ Registry::default()+ .with(RootcauseLayer.with_filter(rust_log_filter))+ .with(fmt_layer).init();+InitializedEntrypoint {tracer_provider: None,}@@ -126,7 +146,10 @@ impl MacroEntrypoint {tree_tracing: Some(level),},) => {- let subscriber = Registry::default().with(HierarchicalLayer::new(level));+ let rust_log_filter = rust_log_env_filter();+ let subscriber = Registry::default()+ .with(RootcauseLayer.with_filter(rust_log_filter.clone()))+ .with(HierarchicalLayer::new(level).with_filter(rust_log_filter));tracing::subscriber::set_global_default(subscriber).unwrap();InitializedEntrypoint {tracer_provider: None,@@ -141,9 +164,13 @@ impl MacroEntrypoint {.unwrap_or_else(|_| "unknown-service".to_string());let tracer = tracer_provider.tracer(service_name);+ let rust_log_filter = rust_log_env_filter();+ let otel_filter = otel_env_filter();+ // Capture anything already enabled for logs or OTEL without enabling new callsites.+ let rootcause_filter = rust_log_filter.clone().or(otel_filter.clone());let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer)- .with_filter(otel_env_filter());+ .with_filter(otel_filter);// Build the JSON event format, then wrap it with DatadogFormat// to inject dd.trace_id / dd.span_id for trace-log correlation.@@ -159,9 +186,13 @@ impl MacroEntrypoint {.with_ansi(false).fmt_fields(tracing_subscriber::fmt::format::JsonFields::new()).event_format(datadog_fmt::DatadogFormat { inner: json_format })- .with_filter(rust_log_env_filter());+ .with_filter(rust_log_filter);- Registry::default().with(fmt_layer).with(otel_layer).init();+ Registry::default()+ .with(RootcauseLayer.with_filter(rootcause_filter))+ .with(fmt_layer)+ .with(otel_layer)+ .init();InitializedEntrypoint {tracer_provider: Some(tracer_provider),diff --git a/crates/macro_entrypoint/src/test.rs b/crates/macro_entrypoint/src/test.rsindex 1f95a8893f..66de4dd691 100644--- a/crates/macro_entrypoint/src/test.rs+++ b/crates/macro_entrypoint/src/test.rs@@ -4,7 +4,7 @@ use opentelemetry::trace::TraceContextExt as _;use opentelemetry::trace::{SpanId, TraceId};use opentelemetry_sdk::trace::InMemorySpanExporter;use std::io;-use std::sync::{Arc, Mutex};+use std::sync::{Arc, Mutex, Once};use tracing_opentelemetry::OpenTelemetrySpanExt as _;use tracing_subscriber::fmt::MakeWriter;@@ -44,14 +44,46 @@ fn test_tracer_provider() -> (InMemorySpanExporter, SdkTracerProvider) {(exporter, provider)}+#[test]+fn rootcause_tracing_subscriber_captures_span_chain() {+ static INSTALL_ROOTCAUSE_HOOKS: Once = Once::new();+ INSTALL_ROOTCAUSE_HOOKS.call_once(install_rootcause_hooks);++ let subscriber = Registry::default().with(RootcauseLayer.with_filter(EnvFilter::new("info")));+ let output = tracing::subscriber::with_default(subscriber, || {+ let request = tracing::info_span!("handle_request", request_id = "req-123");+ let _request_guard = request.enter();+ let document = tracing::info_span!("load_document", document_id = 42);+ let _document_guard = document.enter();++ rootcause::report!("document load failed").to_string()+ });++ assert!(output.contains("document load failed"));+ assert!(output.contains("Tracing spans"));++ let document_span = "load_document{document_id=42}";+ let request_span = "handle_request{request_id=\"req-123\"}";+ assert!(output.contains(document_span), "{output}");+ assert!(output.contains(request_span), "{output}");+ assert!(+ output.find(document_span) < output.find(request_span),+ "span chain should be ordered from the innermost span outward: {output}"+ );+ assert!(!output.contains("Span values missing"), "{output}");+}+#[test]fn rust_log_warn_does_not_drop_info_spans() {let (exporter, provider) = test_tracer_provider();let tracer = provider.tracer("test");+ let rust_log_filter = EnvFilter::new("warn");+ let otel_filter = otel_trace_filter(None);+ let rootcause_filter = rust_log_filter.clone().or(otel_filter.clone());let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer)- .with_filter(otel_trace_filter(None));+ .with_filter(otel_filter);let json_format = tracing_subscriber::fmt::format::Format::default().json()@@ -65,9 +97,12 @@ fn rust_log_warn_does_not_drop_info_spans() {.fmt_fields(tracing_subscriber::fmt::format::JsonFields::new()).event_format(datadog_fmt::DatadogFormat { inner: json_format }).with_writer(logs.clone())- .with_filter(EnvFilter::new("warn"));+ .with_filter(rust_log_filter);- let subscriber = Registry::default().with(fmt_layer).with(otel_layer);+ let subscriber = Registry::default()+ .with(RootcauseLayer.with_filter(rootcause_filter))+ .with(fmt_layer)+ .with(otel_layer);// The span context DatadogFormat reads via Span::current() is only observable here// outside event dispatch: under a scoped test dispatcher, tracing's re-entrancy guard@@ -75,6 +110,10 @@ fn rust_log_warn_does_not_drop_info_spans() {// dispatcher, which get_default resolves without the guard.let mut current_ids: Option<(TraceId, SpanId)> = None;tracing::subscriber::with_default(subscriber, || {+ assert!(tracing::enabled!(tracing::Level::INFO));+ assert!(!tracing::enabled!(tracing::Level::DEBUG));++ tracing::debug_span!("filtered_debug_op").in_scope(|| {});tracing::info_span!("instrumented_op").in_scope(|| {let cx = tracing::Span::current().context();let span_cx = cx.span().span_context().clone();diff --git a/crates/macro_tower_layers/src/lib.rs b/crates/macro_tower_layers/src/lib.rsindex 199910ef21..a6825a0d80 100644--- a/crates/macro_tower_layers/src/lib.rs+++ b/crates/macro_tower_layers/src/lib.rs@@ -4,7 +4,6 @@#[cfg(test)]mod test;use std::{- cmp,sync::{Arc,atomic::{self, AtomicU64},@@ -22,9 +21,9 @@ use tower_http::{ServiceBuilderExt,classify::{ServerErrorsAsFailures, SharedClassifier},request_id::{MakeRequestId, PropagateRequestIdLayer, RequestId, SetRequestIdLayer},- trace::{DefaultMakeSpan, DefaultOnRequest, OnFailure, OnResponse, TraceLayer},+ trace::{MakeSpan, OnFailure, OnResponse, TraceLayer},};-use tracing::{Level, Span};+use tracing::Span;/// A very simple builder for x-request-ids#[derive(Default, Clone)]@@ -41,8 +40,10 @@ impl MakeRequestId for RequestIdBuilder {}}-/// fork of the [DefaultOnResponse] which is deisgined to work with [RequestIdBuilder]-/// This emits a tracing warn event if a request takes more than a certain threshold to complete.+/// Records response telemetry on the request span.+///+/// Successful requests only emit an event when their latency meets or exceeds the warning+/// threshold. Failed requests are logged by [`CustomOnFailure`].#[derive(Clone)]pub struct CustomOnResponse {warning_threshold: Duration,@@ -55,62 +56,53 @@ impl CustomOnResponse {}}-struct Latency {- duration: Duration,+/// Creates INFO-level HTTP server spans with safe OpenTelemetry attributes.+///+/// Request and response headers are intentionally excluded.+#[derive(Clone, Copy, Debug, Default)]+pub struct MakeHttpRequestSpan;++impl<B> MakeSpan<B> for MakeHttpRequestSpan {+ fn make_span(&mut self, request: &Request<B>) -> Span {+ let request_id = request+ .headers()+ .get("x-request-id")+ .and_then(|value| value.to_str().ok())+ .unwrap_or_default();++ tracing::info_span!(+ "http.request",+ otel.kind = "server",+ "http.request.method" = %request.method(),+ "url.path" = request.uri().path(),+ "request.id" = request_id,+ "http.response.status_code" = tracing::field::Empty,+ latency_ms = tracing::field::Empty,+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ )+ }}-impl std::fmt::Display for Latency {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- write!(f, "{} ms", self.duration.as_millis())- }+fn latency_millis(latency: Duration) -> u64 {+ latency.as_millis().try_into().unwrap_or(u64::MAX)}impl<B> OnResponse<B> for CustomOnResponse {fn on_response(self, response: &Response<B>, latency: Duration, span: &Span) {- let level = match latency.cmp(&self.warning_threshold) {- cmp::Ordering::Less => Level::INFO,- cmp::Ordering::Equal | cmp::Ordering::Greater => Level::WARN,- };-- let latency = Latency { duration: latency };-- let response_headers = tracing::field::debug(response.headers());- match level {- Level::ERROR => tracing::error!(+ let status = response.status();+ let latency_ms = latency_millis(latency);+ span.record("http.response.status_code", u64::from(status.as_u16()));+ span.record("latency_ms", latency_ms);++ // Server errors are logged once by CustomOnFailure after this callback returns.+ if !status.is_server_error() && latency >= self.warning_threshold {+ tracing::warn!(parent: span,- %latency,- status = response.status().as_u16(),- response_headers,- "finished processing request"- ),- Level::WARN => tracing::warn!(- parent: span,- %latency,- status = response.status().as_u16(),- response_headers,- "finished processing request"- ),- Level::INFO => tracing::info!(- parent: span,- %latency,- status = response.status().as_u16(),- response_headers,- "finished processing request"- ),- Level::DEBUG => tracing::debug!(- parent: span,- %latency,- status = response.status().as_u16(),- response_headers,- "finished processing request"- ),- Level::TRACE => tracing::trace!(- parent: span,- %latency,- status = response.status().as_u16(),- response_headers,- "finished processing request"- ),+ latency_ms,+ status = status.as_u16(),+ "slow http request"+ );}}}@@ -124,13 +116,19 @@ whereFailureClass: std::fmt::Display,{fn on_failure(&mut self, failure_classification: FailureClass, latency: Duration, span: &Span) {- let latency = Latency { duration: latency };+ let latency_ms = latency_millis(latency);+ span.record("latency_ms", latency_ms);+ span.record("otel.status_code", "ERROR");+ span.record(+ "otel.status_description",+ tracing::field::display(&failure_classification),+ );tracing::error!(parent: span,- classification = %failure_classification,- %latency,- "response failed"+ error = %failure_classification,+ latency_ms,+ "http request failed");}}@@ -141,8 +139,8 @@ type ServiceBuilderAlias = ServiceBuilder<Stack<TraceLayer<SharedClassifier<ServerErrorsAsFailures>,- DefaultMakeSpan,- DefaultOnRequest,+ MakeHttpRequestSpan,+ (),CustomOnResponse,tower_http::trace::DefaultOnBodyChunk,tower_http::trace::DefaultOnEos,@@ -198,7 +196,8 @@ impl MacroRequestIdAndTracingLayer {.set_x_request_id(RequestIdBuilder::default()).layer(TraceLayer::new_for_http()- .make_span_with(DefaultMakeSpan::new().include_headers(true))+ .make_span_with(MakeHttpRequestSpan)+ .on_request(()).on_response(CustomOnResponse::new_with_threshold(warning_threshold)).on_failure(CustomOnFailure),)diff --git a/crates/macro_tower_layers/src/test.rs b/crates/macro_tower_layers/src/test.rsindex ddfbfbbeaf..8c4e9dcce0 100644--- a/crates/macro_tower_layers/src/test.rs+++ b/crates/macro_tower_layers/src/test.rs@@ -1,48 +1,92 @@use super::*;-use std::sync::atomic::{AtomicBool, Ordering};-use tower_http::trace::OnResponse;-use tracing::subscriber::{set_default, with_default};--/// Minimal tracing subscriber that records whether WARN or INFO events were emitted.-struct LevelCapture {- saw_warn: Arc<AtomicBool>,- saw_info: Arc<AtomicBool>,+use std::{+ collections::{HashMap, HashSet},+ sync::{Arc, Mutex},+};+use tower_http::trace::{MakeSpan, OnFailure, OnResponse};+use tracing::{+ Level,+ field::{Field, Visit},+ subscriber::{set_default, with_default},+};++#[derive(Default)]+struct CapturedTracing {+ event_levels: Mutex<Vec<Level>>,+ span_level: Mutex<Option<Level>>,+ declared_span_fields: Mutex<HashSet<String>>,+ initial_span_fields: Mutex<HashMap<String, String>>,+ recorded_span_fields: Mutex<HashMap<String, String>>,}-impl LevelCapture {- fn new() -> (Self, Arc<AtomicBool>, Arc<AtomicBool>) {- let saw_warn = Arc::new(AtomicBool::new(false));- let saw_info = Arc::new(AtomicBool::new(false));+impl CapturedTracing {+ fn event_count(&self, level: Level) -> usize {+ self.event_levels+ .lock()+ .unwrap()+ .iter()+ .filter(|event_level| **event_level == level)+ .count()+ }+}++struct FieldCapture<'a>(&'a mut HashMap<String, String>);++impl Visit for FieldCapture<'_> {+ fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {+ self.0.insert(field.name().to_owned(), format!("{value:?}"));+ }+}++struct TracingCapture {+ captured: Arc<CapturedTracing>,+}++impl TracingCapture {+ fn new() -> (Self, Arc<CapturedTracing>) {+ let captured = Arc::new(CapturedTracing::default());(Self {- saw_warn: saw_warn.clone(),- saw_info: saw_info.clone(),+ captured: captured.clone(),},- saw_warn,- saw_info,+ captured,)}}-impl tracing::Subscriber for LevelCapture {+impl tracing::Subscriber for TracingCapture {fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {true}- fn new_span(&self, _attrs: &tracing::span::Attributes<'_>) -> tracing::span::Id {+ fn new_span(&self, attrs: &tracing::span::Attributes<'_>) -> tracing::span::Id {+ *self.captured.span_level.lock().unwrap() = Some(*attrs.metadata().level());+ *self.captured.declared_span_fields.lock().unwrap() = attrs+ .metadata()+ .fields()+ .iter()+ .map(|field| field.name().to_owned())+ .collect();+ attrs.record(&mut FieldCapture(+ &mut self.captured.initial_span_fields.lock().unwrap(),+ ));tracing::span::Id::from_u64(1)}- fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}+ fn record(&self, _span: &tracing::span::Id, values: &tracing::span::Record<'_>) {+ values.record(&mut FieldCapture(+ &mut self.captured.recorded_span_fields.lock().unwrap(),+ ));+ }fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}fn event(&self, event: &tracing::Event<'_>) {- match *event.metadata().level() {- Level::WARN => self.saw_warn.store(true, Ordering::SeqCst),- Level::INFO => self.saw_info.store(true, Ordering::SeqCst),- _ => {}- }+ self.captured+ .event_levels+ .lock()+ .unwrap()+ .push(*event.metadata().level());}fn enter(&self, _span: &tracing::span::Id) {}@@ -51,52 +95,108 @@ impl tracing::Subscriber for LevelCapture {}#[test]-fn warns_when_latency_exceeds_threshold() {- let (subscriber, saw_warn, saw_info) = LevelCapture::new();+fn request_span_uses_info_and_safe_structured_fields() {+ let (subscriber, captured) = TracingCapture::new();with_default(subscriber, || {- let on_response = CustomOnResponse::new_with_threshold(Duration::from_millis(200));- let response = http::Response::builder().status(200).body(()).unwrap();- let span = tracing::info_span!("test");- on_response.on_response(&response, Duration::from_millis(300), &span);+ let request = Request::builder()+ .method("POST")+ .uri("/documents/123?access_token=secret")+ .header("authorization", "Bearer secret")+ .header("x-request-id", "request-42")+ .body(())+ .unwrap();+ let mut make_span = MakeHttpRequestSpan;+ let _span = make_span.make_span(&request);});- assert!(saw_warn.load(Ordering::SeqCst));- assert!(!saw_info.load(Ordering::SeqCst));+ assert_eq!(*captured.span_level.lock().unwrap(), Some(Level::INFO));++ let declared_fields = captured.declared_span_fields.lock().unwrap();+ assert!(declared_fields.contains("http.request.method"));+ assert!(declared_fields.contains("url.path"));+ assert!(declared_fields.contains("request.id"));+ assert!(declared_fields.contains("http.response.status_code"));+ assert!(declared_fields.contains("latency_ms"));+ assert!(!declared_fields.contains("headers"));+ drop(declared_fields);++ let fields = captured.initial_span_fields.lock().unwrap();+ assert_eq!(fields.get("http.request.method").unwrap(), "POST");+ assert_eq!(fields.get("url.path").unwrap(), "\"/documents/123\"");+ assert_eq!(fields.get("request.id").unwrap(), "\"request-42\"");+ assert!(fields.values().all(|value| !value.contains("secret")));}#[test]-fn info_when_latency_below_threshold() {- let (subscriber, saw_warn, saw_info) = LevelCapture::new();+fn successful_response_records_telemetry_without_completion_event() {+ let (subscriber, captured) = TracingCapture::new();with_default(subscriber, || {- let on_response = CustomOnResponse::new_with_threshold(Duration::from_millis(200));- let response = http::Response::builder().status(200).body(()).unwrap();- let span = tracing::info_span!("test");- on_response.on_response(&response, Duration::from_millis(50), &span);+ let request = Request::builder().uri("/documents").body(()).unwrap();+ let span = MakeHttpRequestSpan.make_span(&request);+ let response = Response::builder().status(204).body(()).unwrap();+ CustomOnResponse::new_with_threshold(Duration::from_millis(200)).on_response(+ &response,+ Duration::from_millis(50),+ &span,+ );});- assert!(!saw_warn.load(Ordering::SeqCst));- assert!(saw_info.load(Ordering::SeqCst));+ assert!(captured.event_levels.lock().unwrap().is_empty());+ let fields = captured.recorded_span_fields.lock().unwrap();+ assert_eq!(fields.get("http.response.status_code").unwrap(), "204");+ assert_eq!(fields.get("latency_ms").unwrap(), "50");}#[test]-fn warns_when_latency_equals_threshold() {- let (subscriber, saw_warn, _saw_info) = LevelCapture::new();+fn slow_response_emits_one_warning() {+ let (subscriber, captured) = TracingCapture::new();with_default(subscriber, || {- let on_response = CustomOnResponse::new_with_threshold(Duration::from_millis(200));- let response = http::Response::builder().status(200).body(()).unwrap();- let span = tracing::info_span!("test");- on_response.on_response(&response, Duration::from_millis(200), &span);+ let request = Request::builder().uri("/documents").body(()).unwrap();+ let span = MakeHttpRequestSpan.make_span(&request);+ let response = Response::builder().status(200).body(()).unwrap();+ CustomOnResponse::new_with_threshold(Duration::from_millis(200)).on_response(+ &response,+ Duration::from_millis(200),+ &span,+ );});- assert!(saw_warn.load(Ordering::SeqCst));+ assert_eq!(captured.event_count(Level::WARN), 1);+ assert_eq!(captured.event_count(Level::INFO), 0);+ assert_eq!(captured.event_count(Level::ERROR), 0);+}++#[test]+fn server_error_emits_only_failure_event() {+ let (subscriber, captured) = TracingCapture::new();++ with_default(subscriber, || {+ let request = Request::builder().uri("/documents").body(()).unwrap();+ let span = MakeHttpRequestSpan.make_span(&request);+ let response = Response::builder().status(500).body(()).unwrap();+ let latency = Duration::from_millis(300);++ CustomOnResponse::new_with_threshold(Duration::from_millis(200))+ .on_response(&response, latency, &span);+ CustomOnFailure.on_failure("Status code: 500", latency, &span);+ });++ assert_eq!(captured.event_count(Level::ERROR), 1);+ assert_eq!(captured.event_count(Level::WARN), 0);+ assert_eq!(captured.event_count(Level::INFO), 0);++ let fields = captured.recorded_span_fields.lock().unwrap();+ assert_eq!(fields.get("http.response.status_code").unwrap(), "500");+ assert_eq!(fields.get("latency_ms").unwrap(), "300");+ assert_eq!(fields.get("otel.status_code").unwrap(), "\"ERROR\"");}#[tokio::test]async fn starvation_detector_warns_when_runtime_blocked() {- let (subscriber, saw_warn, _saw_info) = LevelCapture::new();+ let (subscriber, captured) = TracingCapture::new();let _guard = set_default(subscriber);spawn_starvation_detector(Duration::from_millis(10));@@ -110,14 +210,14 @@ async fn starvation_detector_warns_when_runtime_blocked() {// Let the detector observe the delay and emit the warningtokio::time::sleep(Duration::from_millis(15)).await;- assert!(saw_warn.load(Ordering::SeqCst));+ assert_eq!(captured.event_count(Level::WARN), 1);}#[tokio::test]async fn starvation_detector_does_not_warn_within_grace_period() {tokio::time::pause();- let (subscriber, saw_warn, _saw_info) = LevelCapture::new();+ let (subscriber, captured) = TracingCapture::new();let _guard = set_default(subscriber);spawn_starvation_detector(Duration::from_millis(50));@@ -129,5 +229,5 @@ async fn starvation_detector_does_not_warn_within_grace_period() {tokio::time::advance(Duration::from_millis(54)).await;tokio::task::yield_now().await;- assert!(!saw_warn.load(Ordering::SeqCst));+ assert_eq!(captured.event_count(Level::WARN), 0);}diff --git a/deny.toml b/deny.tomlindex ac10ece054..b9d53a22e5 100644--- a/deny.toml+++ b/deny.toml@@ -204,6 +204,7 @@ deny = ["native_app_server", # Dev tooling only"notification_sandbox", # This is not production code its just tooling"projects", # Dev logging only+ "rootcause-tracing", # External rootcause integration requires tracing-subscriber"sync_service", # Cloudflare/WASM tracing setup uses tracing-web"tracing-opentelemetry", # OpenTelemetry integration requires tracing-subscriber"tracing-panic", # External crate allowed to use tracing-subscriber