Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/feat/improve otel (#5136)

Sean Ayec1b6ae29a60b
diff --git a/Cargo.lock b/Cargo.lock
index 25af734a1e..4aec9f4fb7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8384,6 +8384,8 @@ dependencies = [
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry_sdk",
+ "rootcause",
+ "rootcause-tracing",
"serde_json",
"tracing",
"tracing-opentelemetry",
@@ -12074,6 +12076,17 @@ dependencies = [
"triomphe",
]
+[[package]]
+name = "rootcause-tracing"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3128c7019d8e7814a832aaa712fe55314d1fa0f0ec53e593bc1b4410962db94"
+dependencies = [
+ "rootcause",
+ "tracing",
+ "tracing-subscriber",
+]
+
[[package]]
name = "rowan"
version = "0.16.1"
diff --git a/Cargo.toml b/Cargo.toml
index 7fd9c994fe..e1e8859d4a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -202,6 +202,7 @@ reqwest = { version = "0.13", default-features = false, features = [
] }
rmcp = "1.6.0"
rootcause = { version = "0.12" }
+rootcause-tracing = "0.12.1"
schemars = { version = "1.0.4", features = ["chrono04", "uuid1"] }
semver = { version = "1", features = ["serde"] }
serde = { version = "1.0.214", features = ["derive"] }
diff --git a/crates/macro_entrypoint/Cargo.toml b/crates/macro_entrypoint/Cargo.toml
index 6ef8f1aeb2..3639cb39f3 100644
--- a/crates/macro_entrypoint/Cargo.toml
+++ b/crates/macro_entrypoint/Cargo.toml
@@ -10,6 +10,8 @@ macro_env = { path = "../macro_env" }
opentelemetry = { workspace = true }
opentelemetry_sdk = { workspace = true }
opentelemetry-otlp = { workspace = true }
+rootcause = { workspace = true }
+rootcause-tracing = { workspace = true }
serde_json = { workspace = true }
tracing = { workspace = true }
tracing-opentelemetry = { workspace = true }
diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rs
index 181bbfd504..9329d5a2a9 100644
--- a/crates/macro_entrypoint/src/lib.rs
+++ b/crates/macro_entrypoint/src/lib.rs
@@ -12,8 +12,13 @@ use macro_env_var::{env_vars, maybe_env_vars};
use opentelemetry::trace::TracerProvider as _;
use opentelemetry_otlp::WithExportConfig;
use opentelemetry_sdk::trace::SdkTracerProvider;
+use rootcause::hooks::Hooks;
+use rootcause_tracing::{RootcauseLayer, SpanCollector};
use tracing_subscriber::{
- EnvFilter, Layer, Registry, filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt,
+ EnvFilter, Layer, Registry,
+ filter::{FilterExt, LevelFilter},
+ layer::SubscriberExt,
+ util::SubscriberInitExt,
};
use tracing_tree::HierarchicalLayer;
@@ -56,6 +61,13 @@ fn otel_trace_filter(value: Option<&str>) -> EnvFilter {
.parse_lossy(value.unwrap_or(""))
}
+fn install_rootcause_hooks() {
+ Hooks::new()
+ .report_creation_hook(SpanCollector::new())
+ .install()
+ .expect("failed to install rootcause tracing hooks");
+}
+
/// unit struct which defines the behaviour for instantiation
#[derive(Debug)]
pub struct MacroEntrypoint {
@@ -106,16 +118,24 @@ impl MacroEntrypoint {
pub fn init(self) -> InitializedEntrypoint {
let _ = dotenvy::dotenv();
std::panic::set_hook(Box::new(tracing_panic::panic_hook));
+ install_rootcause_hooks();
match (self.env, self.local) {
(Environment::Local, LocalOptions { tree_tracing: None }) => {
- tracing_subscriber::fmt()
+ // Keep rootcause from enabling callsites that RUST_LOG previously disabled.
+ let rust_log_filter = rust_log_env_filter();
+ let fmt_layer = tracing_subscriber::fmt::layer()
.with_ansi(true)
- .with_env_filter(rust_log_env_filter())
.with_file(true)
.with_line_number(true)
.pretty()
+ .with_filter(rust_log_filter.clone());
+
+ Registry::default()
+ .with(RootcauseLayer.with_filter(rust_log_filter))
+ .with(fmt_layer)
.init();
+
InitializedEntrypoint {
tracer_provider: None,
}
@@ -126,7 +146,10 @@ impl MacroEntrypoint {
tree_tracing: Some(level),
},
) => {
- let subscriber = Registry::default().with(HierarchicalLayer::new(level));
+ let rust_log_filter = rust_log_env_filter();
+ let subscriber = Registry::default()
+ .with(RootcauseLayer.with_filter(rust_log_filter.clone()))
+ .with(HierarchicalLayer::new(level).with_filter(rust_log_filter));
tracing::subscriber::set_global_default(subscriber).unwrap();
InitializedEntrypoint {
tracer_provider: None,
@@ -141,9 +164,13 @@ impl MacroEntrypoint {
.unwrap_or_else(|_| "unknown-service".to_string());
let tracer = tracer_provider.tracer(service_name);
+ let rust_log_filter = rust_log_env_filter();
+ let otel_filter = otel_env_filter();
+ // Capture anything already enabled for logs or OTEL without enabling new callsites.
+ let rootcause_filter = rust_log_filter.clone().or(otel_filter.clone());
let otel_layer = tracing_opentelemetry::layer()
.with_tracer(tracer)
- .with_filter(otel_env_filter());
+ .with_filter(otel_filter);
// Build the JSON event format, then wrap it with DatadogFormat
// to inject dd.trace_id / dd.span_id for trace-log correlation.
@@ -159,9 +186,13 @@ impl MacroEntrypoint {
.with_ansi(false)
.fmt_fields(tracing_subscriber::fmt::format::JsonFields::new())
.event_format(datadog_fmt::DatadogFormat { inner: json_format })
- .with_filter(rust_log_env_filter());
+ .with_filter(rust_log_filter);
- Registry::default().with(fmt_layer).with(otel_layer).init();
+ Registry::default()
+ .with(RootcauseLayer.with_filter(rootcause_filter))
+ .with(fmt_layer)
+ .with(otel_layer)
+ .init();
InitializedEntrypoint {
tracer_provider: Some(tracer_provider),
diff --git a/crates/macro_entrypoint/src/test.rs b/crates/macro_entrypoint/src/test.rs
index 1f95a8893f..66de4dd691 100644
--- a/crates/macro_entrypoint/src/test.rs
+++ b/crates/macro_entrypoint/src/test.rs
@@ -4,7 +4,7 @@ use opentelemetry::trace::TraceContextExt as _;
use opentelemetry::trace::{SpanId, TraceId};
use opentelemetry_sdk::trace::InMemorySpanExporter;
use std::io;
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, Once};
use tracing_opentelemetry::OpenTelemetrySpanExt as _;
use tracing_subscriber::fmt::MakeWriter;
@@ -44,14 +44,46 @@ fn test_tracer_provider() -> (InMemorySpanExporter, SdkTracerProvider) {
(exporter, provider)
}
+#[test]
+fn rootcause_tracing_subscriber_captures_span_chain() {
+ static INSTALL_ROOTCAUSE_HOOKS: Once = Once::new();
+ INSTALL_ROOTCAUSE_HOOKS.call_once(install_rootcause_hooks);
+
+ let subscriber = Registry::default().with(RootcauseLayer.with_filter(EnvFilter::new("info")));
+ let output = tracing::subscriber::with_default(subscriber, || {
+ let request = tracing::info_span!("handle_request", request_id = "req-123");
+ let _request_guard = request.enter();
+ let document = tracing::info_span!("load_document", document_id = 42);
+ let _document_guard = document.enter();
+
+ rootcause::report!("document load failed").to_string()
+ });
+
+ assert!(output.contains("document load failed"));
+ assert!(output.contains("Tracing spans"));
+
+ let document_span = "load_document{document_id=42}";
+ let request_span = "handle_request{request_id=\"req-123\"}";
+ assert!(output.contains(document_span), "{output}");
+ assert!(output.contains(request_span), "{output}");
+ assert!(
+ output.find(document_span) < output.find(request_span),
+ "span chain should be ordered from the innermost span outward: {output}"
+ );
+ assert!(!output.contains("Span values missing"), "{output}");
+}
+
#[test]
fn rust_log_warn_does_not_drop_info_spans() {
let (exporter, provider) = test_tracer_provider();
let tracer = provider.tracer("test");
+ let rust_log_filter = EnvFilter::new("warn");
+ let otel_filter = otel_trace_filter(None);
+ let rootcause_filter = rust_log_filter.clone().or(otel_filter.clone());
let otel_layer = tracing_opentelemetry::layer()
.with_tracer(tracer)
- .with_filter(otel_trace_filter(None));
+ .with_filter(otel_filter);
let json_format = tracing_subscriber::fmt::format::Format::default()
.json()
@@ -65,9 +97,12 @@ fn rust_log_warn_does_not_drop_info_spans() {
.fmt_fields(tracing_subscriber::fmt::format::JsonFields::new())
.event_format(datadog_fmt::DatadogFormat { inner: json_format })
.with_writer(logs.clone())
- .with_filter(EnvFilter::new("warn"));
+ .with_filter(rust_log_filter);
- let subscriber = Registry::default().with(fmt_layer).with(otel_layer);
+ let subscriber = Registry::default()
+ .with(RootcauseLayer.with_filter(rootcause_filter))
+ .with(fmt_layer)
+ .with(otel_layer);
// The span context DatadogFormat reads via Span::current() is only observable here
// outside event dispatch: under a scoped test dispatcher, tracing's re-entrancy guard
@@ -75,6 +110,10 @@ fn rust_log_warn_does_not_drop_info_spans() {
// dispatcher, which get_default resolves without the guard.
let mut current_ids: Option<(TraceId, SpanId)> = None;
tracing::subscriber::with_default(subscriber, || {
+ assert!(tracing::enabled!(tracing::Level::INFO));
+ assert!(!tracing::enabled!(tracing::Level::DEBUG));
+
+ tracing::debug_span!("filtered_debug_op").in_scope(|| {});
tracing::info_span!("instrumented_op").in_scope(|| {
let cx = tracing::Span::current().context();
let span_cx = cx.span().span_context().clone();
diff --git a/crates/macro_tower_layers/src/lib.rs b/crates/macro_tower_layers/src/lib.rs
index 199910ef21..a6825a0d80 100644
--- a/crates/macro_tower_layers/src/lib.rs
+++ b/crates/macro_tower_layers/src/lib.rs
@@ -4,7 +4,6 @@
#[cfg(test)]
mod test;
use std::{
- cmp,
sync::{
Arc,
atomic::{self, AtomicU64},
@@ -22,9 +21,9 @@ use tower_http::{
ServiceBuilderExt,
classify::{ServerErrorsAsFailures, SharedClassifier},
request_id::{MakeRequestId, PropagateRequestIdLayer, RequestId, SetRequestIdLayer},
- trace::{DefaultMakeSpan, DefaultOnRequest, OnFailure, OnResponse, TraceLayer},
+ trace::{MakeSpan, OnFailure, OnResponse, TraceLayer},
};
-use tracing::{Level, Span};
+use tracing::Span;
/// A very simple builder for x-request-ids
#[derive(Default, Clone)]
@@ -41,8 +40,10 @@ impl MakeRequestId for RequestIdBuilder {
}
}
-/// fork of the [DefaultOnResponse] which is deisgined to work with [RequestIdBuilder]
-/// This emits a tracing warn event if a request takes more than a certain threshold to complete.
+/// Records response telemetry on the request span.
+///
+/// Successful requests only emit an event when their latency meets or exceeds the warning
+/// threshold. Failed requests are logged by [`CustomOnFailure`].
#[derive(Clone)]
pub struct CustomOnResponse {
warning_threshold: Duration,
@@ -55,62 +56,53 @@ impl CustomOnResponse {
}
}
-struct Latency {
- duration: Duration,
+/// Creates INFO-level HTTP server spans with safe OpenTelemetry attributes.
+///
+/// Request and response headers are intentionally excluded.
+#[derive(Clone, Copy, Debug, Default)]
+pub struct MakeHttpRequestSpan;
+
+impl<B> MakeSpan<B> for MakeHttpRequestSpan {
+ fn make_span(&mut self, request: &Request<B>) -> Span {
+ let request_id = request
+ .headers()
+ .get("x-request-id")
+ .and_then(|value| value.to_str().ok())
+ .unwrap_or_default();
+
+ tracing::info_span!(
+ "http.request",
+ otel.kind = "server",
+ "http.request.method" = %request.method(),
+ "url.path" = request.uri().path(),
+ "request.id" = request_id,
+ "http.response.status_code" = tracing::field::Empty,
+ latency_ms = tracing::field::Empty,
+ otel.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ )
+ }
}
-impl std::fmt::Display for Latency {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(f, "{} ms", self.duration.as_millis())
- }
+fn latency_millis(latency: Duration) -> u64 {
+ latency.as_millis().try_into().unwrap_or(u64::MAX)
}
impl<B> OnResponse<B> for CustomOnResponse {
fn on_response(self, response: &Response<B>, latency: Duration, span: &Span) {
- let level = match latency.cmp(&self.warning_threshold) {
- cmp::Ordering::Less => Level::INFO,
- cmp::Ordering::Equal | cmp::Ordering::Greater => Level::WARN,
- };
-
- let latency = Latency { duration: latency };
-
- let response_headers = tracing::field::debug(response.headers());
- match level {
- Level::ERROR => tracing::error!(
+ let status = response.status();
+ let latency_ms = latency_millis(latency);
+ span.record("http.response.status_code", u64::from(status.as_u16()));
+ span.record("latency_ms", latency_ms);
+
+ // Server errors are logged once by CustomOnFailure after this callback returns.
+ if !status.is_server_error() && latency >= self.warning_threshold {
+ tracing::warn!(
parent: span,
- %latency,
- status = response.status().as_u16(),
- response_headers,
- "finished processing request"
- ),
- Level::WARN => tracing::warn!(
- parent: span,
- %latency,
- status = response.status().as_u16(),
- response_headers,
- "finished processing request"
- ),
- Level::INFO => tracing::info!(
- parent: span,
- %latency,
- status = response.status().as_u16(),
- response_headers,
- "finished processing request"
- ),
- Level::DEBUG => tracing::debug!(
- parent: span,
- %latency,
- status = response.status().as_u16(),
- response_headers,
- "finished processing request"
- ),
- Level::TRACE => tracing::trace!(
- parent: span,
- %latency,
- status = response.status().as_u16(),
- response_headers,
- "finished processing request"
- ),
+ latency_ms,
+ status = status.as_u16(),
+ "slow http request"
+ );
}
}
}
@@ -124,13 +116,19 @@ where
FailureClass: std::fmt::Display,
{
fn on_failure(&mut self, failure_classification: FailureClass, latency: Duration, span: &Span) {
- let latency = Latency { duration: latency };
+ let latency_ms = latency_millis(latency);
+ span.record("latency_ms", latency_ms);
+ span.record("otel.status_code", "ERROR");
+ span.record(
+ "otel.status_description",
+ tracing::field::display(&failure_classification),
+ );
tracing::error!(
parent: span,
- classification = %failure_classification,
- %latency,
- "response failed"
+ error = %failure_classification,
+ latency_ms,
+ "http request failed"
);
}
}
@@ -141,8 +139,8 @@ type ServiceBuilderAlias = ServiceBuilder<
Stack<
TraceLayer<
SharedClassifier<ServerErrorsAsFailures>,
- DefaultMakeSpan,
- DefaultOnRequest,
+ MakeHttpRequestSpan,
+ (),
CustomOnResponse,
tower_http::trace::DefaultOnBodyChunk,
tower_http::trace::DefaultOnEos,
@@ -198,7 +196,8 @@ impl MacroRequestIdAndTracingLayer {
.set_x_request_id(RequestIdBuilder::default())
.layer(
TraceLayer::new_for_http()
- .make_span_with(DefaultMakeSpan::new().include_headers(true))
+ .make_span_with(MakeHttpRequestSpan)
+ .on_request(())
.on_response(CustomOnResponse::new_with_threshold(warning_threshold))
.on_failure(CustomOnFailure),
)
diff --git a/crates/macro_tower_layers/src/test.rs b/crates/macro_tower_layers/src/test.rs
index ddfbfbbeaf..8c4e9dcce0 100644
--- a/crates/macro_tower_layers/src/test.rs
+++ b/crates/macro_tower_layers/src/test.rs
@@ -1,48 +1,92 @@
use super::*;
-use std::sync::atomic::{AtomicBool, Ordering};
-use tower_http::trace::OnResponse;
-use tracing::subscriber::{set_default, with_default};
-
-/// Minimal tracing subscriber that records whether WARN or INFO events were emitted.
-struct LevelCapture {
- saw_warn: Arc<AtomicBool>,
- saw_info: Arc<AtomicBool>,
+use std::{
+ collections::{HashMap, HashSet},
+ sync::{Arc, Mutex},
+};
+use tower_http::trace::{MakeSpan, OnFailure, OnResponse};
+use tracing::{
+ Level,
+ field::{Field, Visit},
+ subscriber::{set_default, with_default},
+};
+
+#[derive(Default)]
+struct CapturedTracing {
+ event_levels: Mutex<Vec<Level>>,
+ span_level: Mutex<Option<Level>>,
+ declared_span_fields: Mutex<HashSet<String>>,
+ initial_span_fields: Mutex<HashMap<String, String>>,
+ recorded_span_fields: Mutex<HashMap<String, String>>,
}
-impl LevelCapture {
- fn new() -> (Self, Arc<AtomicBool>, Arc<AtomicBool>) {
- let saw_warn = Arc::new(AtomicBool::new(false));
- let saw_info = Arc::new(AtomicBool::new(false));
+impl CapturedTracing {
+ fn event_count(&self, level: Level) -> usize {
+ self.event_levels
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|event_level| **event_level == level)
+ .count()
+ }
+}
+
+struct FieldCapture<'a>(&'a mut HashMap<String, String>);
+
+impl Visit for FieldCapture<'_> {
+ fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
+ self.0.insert(field.name().to_owned(), format!("{value:?}"));
+ }
+}
+
+struct TracingCapture {
+ captured: Arc<CapturedTracing>,
+}
+
+impl TracingCapture {
+ fn new() -> (Self, Arc<CapturedTracing>) {
+ let captured = Arc::new(CapturedTracing::default());
(
Self {
- saw_warn: saw_warn.clone(),
- saw_info: saw_info.clone(),
+ captured: captured.clone(),
},
- saw_warn,
- saw_info,
+ captured,
)
}
}
-impl tracing::Subscriber for LevelCapture {
+impl tracing::Subscriber for TracingCapture {
fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
true
}
- fn new_span(&self, _attrs: &tracing::span::Attributes<'_>) -> tracing::span::Id {
+ fn new_span(&self, attrs: &tracing::span::Attributes<'_>) -> tracing::span::Id {
+ *self.captured.span_level.lock().unwrap() = Some(*attrs.metadata().level());
+ *self.captured.declared_span_fields.lock().unwrap() = attrs
+ .metadata()
+ .fields()
+ .iter()
+ .map(|field| field.name().to_owned())
+ .collect();
+ attrs.record(&mut FieldCapture(
+ &mut self.captured.initial_span_fields.lock().unwrap(),
+ ));
tracing::span::Id::from_u64(1)
}
- fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
+ fn record(&self, _span: &tracing::span::Id, values: &tracing::span::Record<'_>) {
+ values.record(&mut FieldCapture(
+ &mut self.captured.recorded_span_fields.lock().unwrap(),
+ ));
+ }
fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
fn event(&self, event: &tracing::Event<'_>) {
- match *event.metadata().level() {
- Level::WARN => self.saw_warn.store(true, Ordering::SeqCst),
- Level::INFO => self.saw_info.store(true, Ordering::SeqCst),
- _ => {}
- }
+ self.captured
+ .event_levels
+ .lock()
+ .unwrap()
+ .push(*event.metadata().level());
}
fn enter(&self, _span: &tracing::span::Id) {}
@@ -51,52 +95,108 @@ impl tracing::Subscriber for LevelCapture {
}
#[test]
-fn warns_when_latency_exceeds_threshold() {
- let (subscriber, saw_warn, saw_info) = LevelCapture::new();
+fn request_span_uses_info_and_safe_structured_fields() {
+ let (subscriber, captured) = TracingCapture::new();
with_default(subscriber, || {
- let on_response = CustomOnResponse::new_with_threshold(Duration::from_millis(200));
- let response = http::Response::builder().status(200).body(()).unwrap();
- let span = tracing::info_span!("test");
- on_response.on_response(&response, Duration::from_millis(300), &span);
+ let request = Request::builder()
+ .method("POST")
+ .uri("/documents/123?access_token=secret")
+ .header("authorization", "Bearer secret")
+ .header("x-request-id", "request-42")
+ .body(())
+ .unwrap();
+ let mut make_span = MakeHttpRequestSpan;
+ let _span = make_span.make_span(&request);
});
- assert!(saw_warn.load(Ordering::SeqCst));
- assert!(!saw_info.load(Ordering::SeqCst));
+ assert_eq!(*captured.span_level.lock().unwrap(), Some(Level::INFO));
+
+ let declared_fields = captured.declared_span_fields.lock().unwrap();
+ assert!(declared_fields.contains("http.request.method"));
+ assert!(declared_fields.contains("url.path"));
+ assert!(declared_fields.contains("request.id"));
+ assert!(declared_fields.contains("http.response.status_code"));
+ assert!(declared_fields.contains("latency_ms"));
+ assert!(!declared_fields.contains("headers"));
+ drop(declared_fields);
+
+ let fields = captured.initial_span_fields.lock().unwrap();
+ assert_eq!(fields.get("http.request.method").unwrap(), "POST");
+ assert_eq!(fields.get("url.path").unwrap(), "\"/documents/123\"");
+ assert_eq!(fields.get("request.id").unwrap(), "\"request-42\"");
+ assert!(fields.values().all(|value| !value.contains("secret")));
}
#[test]
-fn info_when_latency_below_threshold() {
- let (subscriber, saw_warn, saw_info) = LevelCapture::new();
+fn successful_response_records_telemetry_without_completion_event() {
+ let (subscriber, captured) = TracingCapture::new();
with_default(subscriber, || {
- let on_response = CustomOnResponse::new_with_threshold(Duration::from_millis(200));
- let response = http::Response::builder().status(200).body(()).unwrap();
- let span = tracing::info_span!("test");
- on_response.on_response(&response, Duration::from_millis(50), &span);
+ let request = Request::builder().uri("/documents").body(()).unwrap();
+ let span = MakeHttpRequestSpan.make_span(&request);
+ let response = Response::builder().status(204).body(()).unwrap();
+ CustomOnResponse::new_with_threshold(Duration::from_millis(200)).on_response(
+ &response,
+ Duration::from_millis(50),
+ &span,
+ );
});
- assert!(!saw_warn.load(Ordering::SeqCst));
- assert!(saw_info.load(Ordering::SeqCst));
+ assert!(captured.event_levels.lock().unwrap().is_empty());
+ let fields = captured.recorded_span_fields.lock().unwrap();
+ assert_eq!(fields.get("http.response.status_code").unwrap(), "204");
+ assert_eq!(fields.get("latency_ms").unwrap(), "50");
}
#[test]
-fn warns_when_latency_equals_threshold() {
- let (subscriber, saw_warn, _saw_info) = LevelCapture::new();
+fn slow_response_emits_one_warning() {
+ let (subscriber, captured) = TracingCapture::new();
with_default(subscriber, || {
- let on_response = CustomOnResponse::new_with_threshold(Duration::from_millis(200));
- let response = http::Response::builder().status(200).body(()).unwrap();
- let span = tracing::info_span!("test");
- on_response.on_response(&response, Duration::from_millis(200), &span);
+ let request = Request::builder().uri("/documents").body(()).unwrap();
+ let span = MakeHttpRequestSpan.make_span(&request);
+ let response = Response::builder().status(200).body(()).unwrap();
+ CustomOnResponse::new_with_threshold(Duration::from_millis(200)).on_response(
+ &response,
+ Duration::from_millis(200),
+ &span,
+ );
});
- assert!(saw_warn.load(Ordering::SeqCst));
+ assert_eq!(captured.event_count(Level::WARN), 1);
+ assert_eq!(captured.event_count(Level::INFO), 0);
+ assert_eq!(captured.event_count(Level::ERROR), 0);
+}
+
+#[test]
+fn server_error_emits_only_failure_event() {
+ let (subscriber, captured) = TracingCapture::new();
+
+ with_default(subscriber, || {
+ let request = Request::builder().uri("/documents").body(()).unwrap();
+ let span = MakeHttpRequestSpan.make_span(&request);
+ let response = Response::builder().status(500).body(()).unwrap();
+ let latency = Duration::from_millis(300);
+
+ CustomOnResponse::new_with_threshold(Duration::from_millis(200))
+ .on_response(&response, latency, &span);
+ CustomOnFailure.on_failure("Status code: 500", latency, &span);
+ });
+
+ assert_eq!(captured.event_count(Level::ERROR), 1);
+ assert_eq!(captured.event_count(Level::WARN), 0);
+ assert_eq!(captured.event_count(Level::INFO), 0);
+
+ let fields = captured.recorded_span_fields.lock().unwrap();
+ assert_eq!(fields.get("http.response.status_code").unwrap(), "500");
+ assert_eq!(fields.get("latency_ms").unwrap(), "300");
+ assert_eq!(fields.get("otel.status_code").unwrap(), "\"ERROR\"");
}
#[tokio::test]
async fn starvation_detector_warns_when_runtime_blocked() {
- let (subscriber, saw_warn, _saw_info) = LevelCapture::new();
+ let (subscriber, captured) = TracingCapture::new();
let _guard = set_default(subscriber);
spawn_starvation_detector(Duration::from_millis(10));
@@ -110,14 +210,14 @@ async fn starvation_detector_warns_when_runtime_blocked() {
// Let the detector observe the delay and emit the warning
tokio::time::sleep(Duration::from_millis(15)).await;
- assert!(saw_warn.load(Ordering::SeqCst));
+ assert_eq!(captured.event_count(Level::WARN), 1);
}
#[tokio::test]
async fn starvation_detector_does_not_warn_within_grace_period() {
tokio::time::pause();
- let (subscriber, saw_warn, _saw_info) = LevelCapture::new();
+ let (subscriber, captured) = TracingCapture::new();
let _guard = set_default(subscriber);
spawn_starvation_detector(Duration::from_millis(50));
@@ -129,5 +229,5 @@ async fn starvation_detector_does_not_warn_within_grace_period() {
tokio::time::advance(Duration::from_millis(54)).await;
tokio::task::yield_now().await;
- assert!(!saw_warn.load(Ordering::SeqCst));
+ assert_eq!(captured.event_count(Level::WARN), 0);
}
diff --git a/deny.toml b/deny.toml
index ac10ece054..b9d53a22e5 100644
--- a/deny.toml
+++ b/deny.toml
@@ -204,6 +204,7 @@ deny = [
"native_app_server", # Dev tooling only
"notification_sandbox", # This is not production code its just tooling
"projects", # Dev logging only
+ "rootcause-tracing", # External rootcause integration requires tracing-subscriber
"sync_service", # Cloudflare/WASM tracing setup uses tracing-web
"tracing-opentelemetry", # OpenTelemetry integration requires tracing-subscriber
"tracing-panic", # External crate allowed to use tracing-subscriber