Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
#![deny(missing_docs)]
//! This crate provides a standardized initialization process that should be used across entrypoint crates.
//! This is used to provide consistent behaviour with e.g. tracing configurations
mod datadog_fmt;
mod grafana;
mod shutdown;
#[cfg(test)]
mod test;
pub use shutdown::shutdown_signal;
use macro_env::Environment;
use macro_env_var::{env_vars, maybe_env_vars};
use opentelemetry::trace::TracerProvider as _;
use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
use opentelemetry_otlp::WithExportConfig;
use opentelemetry_sdk::logs::SdkLoggerProvider;
use opentelemetry_sdk::trace::SdkTracerProvider;
use rootcause::hooks::Hooks;
use rootcause_tracing::{RootcauseLayer, SpanCollector};
use tracing_subscriber::{
EnvFilter, Layer, Registry,
filter::{FilterExt, LevelFilter},
layer::SubscriberExt,
util::SubscriberInitExt,
};
use tracing_tree::HierarchicalLayer;
env_vars! {
pub struct DdService;
pub struct DdEnv;
}
maybe_env_vars! {
pub struct RustLog;
pub struct OtelExporterOtlpEndpoint;
pub struct OtelTraceFilter;
pub struct GrafanaOtlpEndpoint;
}
/// Build an [`EnvFilter`] from `RUST_LOG`, honoring values injected via `APP_SECRETS_JSON`.
///
/// [`EnvFilter::from_default_env`] only reads the process environment, so a `RUST_LOG` set through
/// `APP_SECRETS_JSON` (the way the rest of our config is injected, see `macro_env_var`) is ignored
/// and our tracing filter ends up wrong. This reads `RUST_LOG` the same way `macro_env_var` does —
/// `APP_SECRETS_JSON` first, then the process environment as a fallback.
/// Both branches carry an INFO floor, for the same reason
/// [`otel_trace_filter`] does. A builder with no default directive and an
/// absent or empty `RUST_LOG` yields a filter with *zero* directives, which
/// logs nothing at all - not even ERROR. A service deployed without an
/// explicit `RUST_LOG` is then silent, while its spans keep flowing, so it
/// reads as healthy-but-quiet rather than as misconfigured.
fn rust_log_env_filter() -> EnvFilter {
match RustLog::new() {
Some(value) => rust_log_filter(Some(&value)),
// `from_env_lossy` reads `RUST_LOG` itself, so the process
// environment stays behind the builder rather than a direct
// `std::env::var` (CS-14).
None => rust_log_builder().from_env_lossy(),
}
}
fn rust_log_builder() -> tracing_subscriber::filter::Builder {
EnvFilter::builder().with_default_directive(LevelFilter::INFO.into())
}
fn rust_log_filter(value: Option<&str>) -> EnvFilter {
rust_log_builder().parse_lossy(value.unwrap_or(""))
}
/// Build an [`EnvFilter`] for the OpenTelemetry span exporter from `OTEL_TRACE_FILTER`.
///
/// Traces are filtered independently of `RUST_LOG` so that lowering log verbosity (e.g.
/// `RUST_LOG=warn`) cannot silence APM traces — `#[tracing::instrument]` spans default to INFO
/// and a global filter would drop them before the otel layer sees them. Defaults to `info` when
/// `OTEL_TRACE_FILTER` is unset or contains no valid directives.
fn otel_env_filter() -> EnvFilter {
otel_trace_filter(OtelTraceFilter::new().as_deref())
}
fn otel_trace_filter(value: Option<&str>) -> EnvFilter {
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.parse_lossy(value.unwrap_or(""))
}
fn install_rootcause_hooks() {
Hooks::new()
.report_creation_hook(SpanCollector::new())
.install()
.expect("failed to install rootcause tracing hooks");
}
/// unit struct which defines the behaviour for instantiation
#[derive(Debug)]
pub struct MacroEntrypoint {
env: Environment,
/// describes options that only apply in local dev
local: LocalOptions,
/// Dependency targets whose instrumentation can include credentials or application payloads.
suppressed_targets: &'static [&'static str],
}
impl Default for MacroEntrypoint {
fn default() -> Self {
// Load .env file if it exists, but don't fail if it doesn't
let _ = dotenvy::dotenv();
MacroEntrypoint {
env: Environment::new_or_prod(),
local: Default::default(),
suppressed_targets: &[],
}
}
}
/// sentinel struct which guarantees that we called [MacroEntrypoint::init]
#[derive(Debug)]
pub struct InitializedEntrypoint {
tracer_provider: Option<SdkTracerProvider>,
logger_provider: Option<SdkLoggerProvider>,
}
/// Failure to flush telemetry before a serverless runtime freezes.
#[derive(Debug, thiserror::Error)]
pub enum TelemetryFlushError {
/// The blocking flush task could not finish.
#[error("telemetry flush task failed")]
Task(#[from] tokio::task::JoinError),
/// The trace exporter could not flush.
#[error("trace flush failed")]
Traces(#[source] opentelemetry_sdk::error::OTelSdkError),
/// The log exporter could not flush.
#[error("log flush failed")]
Logs(#[source] opentelemetry_sdk::error::OTelSdkError),
}
impl InitializedEntrypoint {
/// Flush completed telemetry before a serverless runtime freezes, without
/// shutting down providers needed by subsequent warm invocations.
///
/// Run the blocking SDK flush off the async executor so gRPC can progress
/// even on a single-threaded runtime. Call after invocation spans close.
pub async fn force_flush(&self) -> Result<(), TelemetryFlushError> {
let tracer = self.tracer_provider.clone();
let logger = self.logger_provider.clone();
let (traces, logs) = tokio::task::spawn_blocking(move || {
(
tracer.map(|provider| provider.force_flush()),
logger.map(|provider| provider.force_flush()),
)
})
.await?;
traces.transpose().map_err(TelemetryFlushError::Traces)?;
logs.transpose().map_err(TelemetryFlushError::Logs)?;
Ok(())
}
/// Gracefully shut down the OpenTelemetry providers.
/// This should be called before the application exits to ensure all traces and logs are flushed.
pub fn shutdown(&self) {
if let Some(ref provider) = self.tracer_provider
&& let Err(e) = provider.shutdown()
{
tracing::error!(error=?e, "failed to shutdown tracer provider");
}
if let Some(ref provider) = self.logger_provider
&& let Err(e) = provider.shutdown()
{
tracing::error!(error=?e, "failed to shutdown logger provider");
}
}
}
impl MacroEntrypoint {
/// Suppress credential-bearing dependency diagnostics in every sink, even if RUST_LOG enables trace.
pub fn suppress_dependency_logs(mut self, targets: &'static [&'static str]) -> Self {
self.suppressed_targets = targets;
self
}
/// create a new instance of [Self] from an input [Environment]
pub fn new(env: Environment) -> Self {
Self {
env,
..Default::default()
}
}
/// consume self, initialize this binary, and return a proof that it was initialized [InitializedEntrypoint]
pub fn init(self) -> InitializedEntrypoint {
let _ = dotenvy::dotenv();
std::panic::set_hook(Box::new(tracing_panic::panic_hook));
install_rootcause_hooks();
match (self.env, self.local) {
(Environment::Local, LocalOptions { tree_tracing: None }) => {
let rust_log_filter = rust_log_env_filter();
// Local OTLP export is opt-in. xtask injects this only for
// trace-enabled local runs. Alongside spans, tracing events
// ship as OTLP log records (Loki, when the collector is the
// LGTM stack), so local logs are queryable next to the traces
// they belong to.
let export_otel = OtelExporterOtlpEndpoint::new().is_some();
let tracer_provider =
export_otel.then(|| init_opentelemetry("local".to_string(), None));
let logger_provider = export_otel.then(|| init_otel_logs("local".to_string()));
if let Some(provider) = tracer_provider.as_ref() {
let otel_filter = otel_env_filter();
let rootcause_filter = rust_log_filter.clone().or(otel_filter.clone());
let otel_layer = otel_layer_with_error_mapping(provider.tracer(service_name()))
.with_filter(otel_filter);
let log_bridge = logger_provider.as_ref().map(|p| {
OpenTelemetryTracingBridge::new(p).with_filter(otel_logs_filter())
});
Registry::default()
.with(tracing_subscriber::filter::filter_fn(move |metadata| {
dependency_logs_allowed(self.suppressed_targets, metadata.target())
}))
.with(RootcauseLayer.with_filter(rootcause_filter))
.with(
tracing_subscriber::fmt::layer()
.with_ansi(true)
.with_file(true)
.with_line_number(true)
.pretty()
.with_filter(rust_log_filter),
)
.with(otel_layer)
.with(log_bridge)
.init();
} else {
Registry::default()
.with(tracing_subscriber::filter::filter_fn(move |metadata| {
dependency_logs_allowed(self.suppressed_targets, metadata.target())
}))
.with(RootcauseLayer.with_filter(rust_log_filter.clone()))
.with(
tracing_subscriber::fmt::layer()
.with_ansi(true)
.with_file(true)
.with_line_number(true)
.pretty()
.with_filter(rust_log_filter),
)
.init();
}
InitializedEntrypoint {
tracer_provider,
logger_provider,
}
}
(
Environment::Local,
LocalOptions {
tree_tracing: Some(level),
},
) => {
let rust_log_filter = rust_log_env_filter();
let subscriber = Registry::default()
.with(tracing_subscriber::filter::filter_fn(move |metadata| {
dependency_logs_allowed(self.suppressed_targets, metadata.target())
}))
.with(RootcauseLayer.with_filter(rust_log_filter.clone()))
.with(HierarchicalLayer::new(level).with_filter(rust_log_filter));
tracing::subscriber::set_global_default(subscriber).unwrap();
InitializedEntrypoint {
tracer_provider: None,
logger_provider: None,
}
}
(Environment::Production | Environment::Develop, _) => {
// Get environment from DD_ENV
let env = DdEnv::new()
.map(|e| e.to_string())
.unwrap_or_else(|_| "unknown".to_string());
let grafana_endpoint = matches!(self.env, Environment::Develop)
.then(GrafanaOtlpEndpoint::new)
.flatten();
let tracer_provider = init_opentelemetry(env, grafana_endpoint.as_deref());
let tracer = tracer_provider.tracer(service_name());
let rust_log_filter = rust_log_env_filter();
let otel_filter = otel_env_filter();
// Capture anything already enabled for logs or OTEL without enabling new callsites.
let rootcause_filter = rust_log_filter.clone().or(otel_filter.clone());
let otel_layer = otel_layer_with_error_mapping(tracer).with_filter(otel_filter);
// Build the JSON event format, then wrap it with DatadogFormat
// to inject dd.trace_id / dd.span_id for trace-log correlation.
let json_format = tracing_subscriber::fmt::format::Format::default()
.json()
.with_current_span(true)
.with_span_list(false)
.flatten_event(true)
.with_file(true)
.with_line_number(true);
let fmt_layer = tracing_subscriber::fmt::layer()
.with_ansi(false)
.fmt_fields(tracing_subscriber::fmt::format::JsonFields::new())
.event_format(datadog_fmt::DatadogFormat { inner: json_format })
.with_filter(rust_log_filter);
Registry::default()
.with(tracing_subscriber::filter::filter_fn(move |metadata| {
dependency_logs_allowed(self.suppressed_targets, metadata.target())
}))
.with(RootcauseLayer.with_filter(rootcause_filter))
.with(fmt_layer)
.with(otel_layer)
.init();
// Prod/develop logs reach Datadog as JSON on stdout (the fmt
// layer above); no OTLP log export there.
InitializedEntrypoint {
tracer_provider: Some(tracer_provider),
logger_provider: None,
}
}
}
}
/// begin modifying the options for the local environment
pub fn local(self) -> LocalOptionsBuilder {
LocalOptionsBuilder {
prev: self,
next: Default::default(),
}
}
}
/// Default OpenTelemetry export endpoint: the Datadog agent sidecar in prod/develop.
const DEFAULT_OTLP_ENDPOINT: &str = "http://127.0.0.1:4317";
/// Wraps the OTel layer so `tracing::error!` events on a span (e.g.
/// `macro_tower_layers::CustomOnFailure`'s "response failed", or an
/// `#[instrument(err)]` failure) become an OTLP exception with a message and
/// stack trace, and set the span's error status message. Without these, a
/// span can be marked errored (from the HTTP status) with no `error.message`
/// / `error.stack` for Datadog to show — "Missing error message and stack
/// trace" in the UI.
fn otel_layer_with_error_mapping<S>(
tracer: opentelemetry_sdk::trace::SdkTracer,
) -> tracing_opentelemetry::OpenTelemetryLayer<S, opentelemetry_sdk::trace::SdkTracer>
where
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
{
tracing_opentelemetry::layer()
.with_tracer(tracer)
.with_error_events_to_exceptions(true)
.with_error_events_to_status(true)
.with_error_fields_to_exceptions(true)
.with_error_records_to_exceptions(true)
.with_location(true)
}
/// The service name reported on traces: `DD_SERVICE` when set, otherwise the
/// current executable's file name so each service is distinguishable locally.
fn service_name() -> String {
if let Ok(service) = DdService::new() {
return service.to_string();
}
std::env::current_exe()
.ok()
.and_then(|exe| {
exe.file_name()
.map(|name| name.to_string_lossy().into_owned())
})
.unwrap_or_else(|| "unknown-service".to_string())
}
fn otel_endpoint() -> String {
OtelExporterOtlpEndpoint::new()
.map(|e| e.to_string())
.unwrap_or_else(|| DEFAULT_OTLP_ENDPOINT.to_string())
}
fn otel_resource(deployment_environment: String) -> opentelemetry_sdk::Resource {
opentelemetry_sdk::Resource::builder()
.with_service_name(service_name())
.with_attribute(opentelemetry::KeyValue::new(
"deployment.environment",
deployment_environment,
))
.build()
}
fn init_opentelemetry(
deployment_environment: String,
grafana_endpoint: Option<&str>,
) -> SdkTracerProvider {
// W3C trace-context propagation: lets macro_tower_layers parent request
// spans under an incoming `traceparent` (e.g. from the web app), and
// service clients propagate context onward.
opentelemetry::global::set_text_map_propagator(
opentelemetry_sdk::propagation::TraceContextPropagator::new(),
);
let exporter = opentelemetry_otlp::SpanExporter::builder()
.with_tonic()
.with_endpoint(otel_endpoint())
.build()
.expect("failed to create OTLP span exporter");
let mut provider = SdkTracerProvider::builder()
.with_batch_exporter(exporter)
.with_resource(otel_resource(deployment_environment));
// Keep the original exporter and its queue independent. The second batch
// processor copies the same finished spans without changing IDs or sampling.
if let Some(endpoint) = grafana_endpoint.filter(|endpoint| !endpoint.trim().is_empty()) {
match opentelemetry_otlp::SpanExporter::builder()
.with_tonic()
.with_endpoint(endpoint)
.with_timeout(std::time::Duration::from_secs(2))
.build()
{
Ok(exporter) => {
// At most one in-flight and one queued batch: two two-second
// exports fit within the SDK's five-second shutdown timeout.
let processor = opentelemetry_sdk::trace::BatchSpanProcessor::builder(
grafana::GrafanaSpanExporter(exporter),
)
.with_batch_config(
opentelemetry_sdk::trace::BatchConfigBuilder::default()
.with_max_queue_size(512)
.with_max_export_batch_size(512)
.build(),
)
.build();
provider = provider.with_span_processor(processor);
}
// Tracing is not installed yet. Optional export must not prevent startup.
Err(error) => eprintln!("Grafana trace exporter unavailable: {error}"),
}
}
provider.build()
}
/// OTLP log export for the [`OpenTelemetryTracingBridge`]: tracing events
/// become log records (with trace/span correlation) at the same endpoint the
/// spans go to. Local-only today — see the prod arm of [`MacroEntrypoint::init`].
fn init_otel_logs(deployment_environment: String) -> SdkLoggerProvider {
let exporter = opentelemetry_otlp::LogExporter::builder()
.with_tonic()
.with_endpoint(otel_endpoint())
.build()
.expect("failed to create OTLP log exporter");
SdkLoggerProvider::builder()
.with_batch_exporter(exporter)
.with_resource(otel_resource(deployment_environment))
.build()
}
/// Filter for the log bridge: mirror `RUST_LOG` verbosity, but never feed the
/// exporter's own internals back into it — the OTel/tonic stack logs through
/// `tracing`, so exporting those events would emit more of them (a
/// telemetry-induced-telemetry loop).
fn otel_logs_filter() -> EnvFilter {
let mut filter = rust_log_env_filter();
for directive in [
"opentelemetry=off",
"opentelemetry_sdk=off",
"opentelemetry_otlp=off",
"tonic=off",
"h2=off",
"hyper=off",
] {
filter = filter.add_directive(directive.parse().expect("static directive parses"));
}
filter
}
/// builder struct for modifying the local environment options
pub struct LocalOptionsBuilder {
prev: MacroEntrypoint,
next: LocalOptions,
}
impl LocalOptionsBuilder {
/// change the options for tree_tracing
pub fn tree_tracing(mut self, val: Option<usize>) -> Self {
self.next.tree_tracing = val;
self
}
/// finish modifying the options for the local environment
pub fn build(self) -> MacroEntrypoint {
MacroEntrypoint {
env: self.prev.env,
local: self.next,
suppressed_targets: self.prev.suppressed_targets,
}
}
}
#[derive(Debug, Default)]
struct LocalOptions {
tree_tracing: Option<usize>,
}
fn dependency_logs_allowed(suppressed: &[&str], target: &str) -> bool {
!suppressed.iter().any(|prefix| {
target == *prefix
|| target
.strip_prefix(prefix)
.is_some_and(|suffix| suffix.starts_with("::"))
})
}