Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

49.8 KB
Raw Blame
use std::collections::HashMap;
use std::num::NonZeroU32;
use std::sync::Arc;
use std::time::Duration;

use anyhow::Context;
use axum::Router;
use common_ingestion_warnings::{
    observe_delivery, KafkaWarningEmitter, WarningEmitter, WarningThrottle,
    INGESTION_WARNINGS_EMITTER_ENABLED,
};
use common_kafka::config::KafkaConfig as WarningsKafkaConfig;
use common_kafka::kafka_producer::create_threaded_kafka_producer_no_ping;
use common_redis::RedisClient;
use metrics::gauge;
use tracing::{info, warn};

use crate::config::{CaptureMode, Config};
use crate::event_restrictions::{EventRestrictionService, Pipeline, RedisRestrictionsRepository};
use crate::global_rate_limiter::{ai_byte_limit_window, GlobalRateLimiter};
use crate::outputs::{Output, OutputRegistry};
use crate::producers::{self, ProducerConfig, ProducerName, ProducerRegistry};
use crate::prometheus::setup_metrics_recorder;
use crate::quota_limiters::{
    is_exception_event, is_llm_event, is_survey_event, CaptureQuotaLimiter,
};
use crate::router;
use crate::router::BATCH_BODY_SIZE;
use crate::sinks::kafka::KafkaSink;
use crate::sinks::noop::NoOpSink;
use crate::sinks::print::PrintSink;
use crate::sinks::registry::OutputTable;
use crate::sinks::s3::S3Sink;
use limiters::overflow::OverflowLimiter;
use limiters::redis::{QuotaResource, RedisLimiter, ServiceName, OVERFLOW_LIMITER_CACHE_KEY};
use limiters::token_dropper::TokenDropper;

pub struct LifecycleHandles {
    pub server: lifecycle::Handle,
    pub sink: Option<lifecycle::Handle>,
    pub advisory: Option<lifecycle::Handle>,
    pub event_restrictions: Option<lifecycle::Handle>,
    pub ingestion_warnings: Option<lifecycle::Handle>,
    pub readiness: lifecycle::ReadinessHandler,
    pub liveness: lifecycle::LivenessHandler,
}

pub fn register_components(manager: &mut lifecycle::Manager, config: &Config) -> LifecycleHandles {
    let server = manager.register(
        "server",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(60)),
    );

    let sink_opts =
        lifecycle::ComponentOptions::new().with_liveness_deadline(Duration::from_secs(30));

    let (sink, advisory) = if config.print_sink || config.noop_sink {
        (None, None)
    } else if config.s3_fallback_enabled {
        let kafka = manager.register("kafka-sink", sink_opts.clone().is_advisory(true));
        let s3 = manager.register("s3-sink", sink_opts.clone());
        (Some(s3), Some(kafka))
    } else {
        (
            Some(manager.register("kafka-sink", sink_opts.clone())),
            None,
        )
    };

    let event_restrictions =
        if config.event_restrictions_enabled && config.event_restrictions_redis_url.is_some() {
            Some(manager.register("event-restrictions", lifecycle::ComponentOptions::new()))
        } else {
            None
        };

    // Advisory: the warnings producer is best-effort, so a stalled or dead
    // producer must never gate pod liveness/readiness or trigger shutdown.
    let ingestion_warnings = if config.capture_ingestion_warnings_enabled {
        Some(
            manager.register(
                "ingestion-warnings",
                lifecycle::ComponentOptions::new()
                    .with_liveness_deadline(Duration::from_secs(30))
                    .is_advisory(true),
            ),
        )
    } else {
        None
    };

    let readiness = manager.readiness_handler();
    let liveness = manager.liveness_handler();

    LifecycleHandles {
        server,
        sink,
        advisory,
        event_restrictions,
        ingestion_warnings,
        readiness,
        liveness,
    }
}

pub struct CaptureComponents {
    pub app: Router,
    pub server_handle: lifecycle::Handle,
    /// `None` with the print or noop sink, which produce to no Kafka.
    pub producers: Option<Arc<ProducerRegistry>>,
    pub event_restriction_service: Option<EventRestrictionService>,
    pub http1_header_read_timeout_ms: Option<u64>,
}

pub async fn build_components(
    config: Config,
    env: HashMap<String, String>,
    handles: LifecycleHandles,
) -> CaptureComponents {
    let LifecycleHandles {
        server,
        sink: sink_handle,
        advisory: advisory_handle,
        event_restrictions: event_restrictions_handle,
        ingestion_warnings: ingestion_warnings_handle,
        readiness,
        liveness,
    } = handles;

    // Must come first: metrics emitted before the global recorder exists are
    // silently dropped, and its `role`/`capture_mode` labels are fixed here.
    let recorder_handle = config.export_prometheus.then(|| {
        setup_metrics_recorder(
            config.otel_service_name.clone(),
            config.capture_mode.as_tag(),
        )
    });

    let producer_configs = producers::load_all(&env)
        .unwrap_or_else(|e| panic!("fatal: invalid producer configuration: {e:#}"));
    let ingestion_message_max_bytes = producer_configs[&ProducerName::Ingestion].message_max_bytes;

    let redis_client = Arc::new(
        RedisClient::with_config(
            config.redis_url.clone(),
            common_redis::CompressionConfig::disabled(),
            common_redis::RedisValueFormat::default(),
            if config.redis_response_timeout_ms == 0 {
                None
            } else {
                Some(Duration::from_millis(config.redis_response_timeout_ms))
            },
            if config.redis_connection_timeout_ms == 0 {
                None
            } else {
                Some(Duration::from_millis(config.redis_connection_timeout_ms))
            },
        )
        .await
        .expect("failed to create redis client"),
    );

    // With GLOBAL_RATE_LIMIT_REDIS_URL set, each limiter gets its own client so a slow
    // drain on one cannot eat the other's timeout budget; without it, both share the main client.
    // Only the per-(token, distinct_id) limiter shards across GLOBAL_RATE_LIMIT_REDIS_SHARDS clusters.
    let ai_byte_limit_enabled = ai_byte_limit_per_second(&config) > 0;
    let rate_limiter_redis = if config.global_rate_limit_enabled {
        Some(
            GlobalRateLimiter::build_redis_shards(&config, redis_client.clone())
                .await
                .expect("failed to create rate limiter redis client"),
        )
    } else {
        None
    };
    let ai_byte_limiter_redis = if ai_byte_limit_enabled {
        Some(
            GlobalRateLimiter::build_redis_client(&config, redis_client.clone())
                .await
                .expect("failed to create AI byte limiter redis client"),
        )
    } else {
        None
    };

    // The dynamic custom-threshold refresh loop is owned by the common limiter:
    // when GLOBAL_RATE_LIMIT_CUSTOM_THRESHOLD_KEY is set, `build()` wires a Redis
    // source into the limiter, which spawns and manages the refresh task itself.
    let global_rate_limiter_token_distinctid = rate_limiter_redis.as_ref().map(|redis| {
        Arc::new(
            GlobalRateLimiter::new_token_distinct_id(&config, redis.clone())
                .expect("failed to create global rate limiter"),
        )
    });

    // add new "scoped" quota limiters here as new quota tracking buckets are added
    // to PostHog! Here a "scoped" limiter is one that should be INDEPENDENT of the
    // global billing limiter applied here to every event batch. You must supply the
    // QuotaResource type and a predicate function that will match events to be limited
    let quota_limiter =
        CaptureQuotaLimiter::new(&config, redis_client.clone(), Duration::from_secs(5))
            .add_scoped_limiter(QuotaResource::Exceptions, is_exception_event)
            .add_scoped_limiter(QuotaResource::Surveys, is_survey_event)
            .add_scoped_limiter(QuotaResource::LLMEvents, is_llm_event);

    // TODO: remove this once we have a billing limiter
    let token_dropper = config
        .drop_events_by_token_distinct_id
        .clone()
        .map(|k| TokenDropper::new(&k))
        .unwrap_or_default();

    // In Recordings capture mode, we unpack a batch of events, and then pack them back up into
    // a big blob and send to kafka all at once - so we should abort unpacking a batch if the data
    // size crosses the kafka limit. In the Events mode, we can unpack the batch and send each
    // event individually, so we should instead allow for some small multiple of our max compressed
    // body size to be unpacked. If a single event is still too big, we'll drop it at kafka send time.
    let event_payload_max_bytes = match config.capture_mode {
        CaptureMode::Events | CaptureMode::Ai | CaptureMode::Import => BATCH_BODY_SIZE * 5,
        CaptureMode::Recordings => ingestion_message_max_bytes as usize,
    };

    // Build the overflow limiters here (not inside the sink) so routing
    // policy lives in `router::State` alongside every other pipeline-level
    // decision. The kafka sink used to own these; after the refactor it is
    // a pure mechanism layer and reads `metadata.overflow_reason` that the
    // pipeline stamps upstream. See `router::State::overflow_limiter` and
    // `router::State::replay_overflow_limiter`.
    let overflow_limiter: Option<Arc<OverflowLimiter>> = if config.overflow_enabled {
        let partition = OverflowLimiter::new(
            config.overflow_per_second_limit,
            config.overflow_burst_limit,
            config.ingestion_force_overflow_by_token_distinct_id.clone(),
            config.overflow_preserve_partition_locality,
        );

        if config.export_prometheus {
            let partition = partition.clone();
            tokio::spawn(async move {
                partition.report_metrics("analytics").await;
            });
        }

        {
            // Keep the governor's per-key state from growing unbounded.
            let partition = partition.clone();
            tokio::spawn(async move {
                partition.clean_state().await;
            });
        }

        Some(Arc::new(partition))
    } else {
        None
    };

    let replay_overflow_limiter: Option<Arc<RedisLimiter>> = match config.capture_mode {
        CaptureMode::Recordings => Some(Arc::new(
            RedisLimiter::new(
                Duration::from_secs(5),
                redis_client.clone(),
                OVERFLOW_LIMITER_CACHE_KEY.to_string(),
                config.redis_key_prefix.clone(),
                QuotaResource::Replay,
                ServiceName::Capture,
            )
            .expect("failed to start replay overflow limiter"),
        )),
        _ => None,
    };

    let (outputs, producers) =
        create_output_registry(&config, &producer_configs, sink_handle, advisory_handle)
            .await
            .expect("failed to create the output registry");
    let outputs = Arc::new(outputs);

    let event_restriction_service = if let Some(handle) = event_restrictions_handle {
        create_event_restriction_service(
            &config,
            handle,
            Pipeline::for_capture_mode(config.capture_mode),
        )
    } else {
        None
    };

    assert!(
        !config.outputs.ai_main_topic.is_empty(),
        "invalid configuration: CAPTURE_OUTPUT_AI_MAIN_TOPIC must not be empty",
    );
    let ai_events_overflow_enabled = ai_events_overflow_limiter_enabled(&config);
    info!(
        ai_main_topic = %config.outputs.ai_main_topic,
        ai_overflow_topic = %config.outputs.ai_overflow_topic,
        ai_events_overflow_enabled,
        "AI events topic routing"
    );

    // The AI lane gets its own limiter instance with the same knobs: the
    // governor state (per-`token:distinct_id` budgets) is what must stay
    // isolated, so analytics volume can never push a key's AI events into
    // AI overflow and AI volume never burns the analytics budget.
    let ai_events_overflow_limiter: Option<Arc<OverflowLimiter>> =
        if config.overflow_enabled && ai_events_overflow_enabled {
            let limiter = OverflowLimiter::new(
                config.overflow_per_second_limit,
                config.overflow_burst_limit,
                config.ingestion_force_overflow_by_token_distinct_id.clone(),
                config.overflow_preserve_partition_locality,
            );

            if config.export_prometheus {
                let limiter = limiter.clone();
                tokio::spawn(async move {
                    limiter.report_metrics("ai").await;
                });
            }

            {
                // Keep the governor's per-key state from growing unbounded.
                let limiter = limiter.clone();
                tokio::spawn(async move {
                    limiter.clean_state().await;
                });
            }

            Some(Arc::new(limiter))
        } else {
            None
        };

    // Unlike the governor-backed overflow limiters above, this one needs no
    // metrics or state-cleanup tasks of its own: the global rate limiter owns
    // its background tick loop, its cache eviction, and its own metric series
    // (scoped `<mode>_ai_bytes`).
    if ai_byte_limit_enabled {
        warn_if_ai_byte_budget_below_max_event(&config);
    }
    warn_if_ai_ceiling_exceeds_producer_cap(&config, ingestion_message_max_bytes);
    let ai_byte_rate_limiter = ai_byte_limiter_redis.as_ref().map(|redis| {
        Arc::new(
            GlobalRateLimiter::new_ai_bytes(&config, vec![redis.clone()])
                .expect("failed to create AI byte rate limiter"),
        )
    });

    let ingestion_warning_emitter =
        create_ingestion_warning_emitter(&config, ingestion_warnings_handle).await;

    let app = router::router(
        crate::time::SystemTime {},
        readiness,
        liveness,
        outputs,
        redis_client,
        global_rate_limiter_token_distinctid,
        quota_limiter,
        token_dropper,
        event_restriction_service.clone(),
        recorder_handle,
        config.capture_mode,
        config.concurrency_limit,
        event_payload_max_bytes,
        config.enable_historical_rerouting,
        config.historical_rerouting_threshold_days,
        config.is_mirror_deploy,
        config.verbose_sample_percent,
        config.ai_max_sum_of_parts_bytes,
        config.ai_max_event_bytes,
        config.body_chunk_read_timeout_ms,
        config.body_read_chunk_size_kb,
        config.capture_v1_max_compressed_body_bytes,
        config.capture_v1_max_decompressed_body_bytes,
        overflow_limiter,
        ai_events_overflow_limiter,
        ai_byte_rate_limiter,
        replay_overflow_limiter,
        config.capture_v1_scatter_gather_min_batch,
        config.ai_gateway_signing_secret.clone(),
        ingestion_warning_emitter,
    );

    info!(
        "config: is_mirror_deploy == {:?} ; log_level == {:?}",
        config.is_mirror_deploy, config.log_level
    );

    CaptureComponents {
        app,
        server_handle: server,
        producers: producers.map(Arc::new),
        event_restriction_service,
        http1_header_read_timeout_ms: config.http1_header_read_timeout_ms,
    }
}

fn ai_events_overflow_limiter_enabled(config: &Config) -> bool {
    let enabled = config.ai_events_overflow_enabled;
    assert!(
        !(enabled && matches!(config.capture_mode, CaptureMode::Import)),
        "invalid configuration: AI_EVENTS_OVERFLOW_ENABLED must be false in import mode; imports must never overflow"
    );
    enabled
}

/// The AI byte budget this deployment enforces, or `0` to skip building the
/// limiter entirely. Import is exempt — backfills are never throttled, matching
/// the other limiters — and not building the limiter is the whole exemption, so
/// neither pipeline's charge step needs capture-mode awareness of its own.
fn ai_byte_limit_per_second(config: &Config) -> u64 {
    if matches!(config.capture_mode, CaptureMode::Import) {
        return 0;
    }
    config.ai_byte_limit_per_second
}

/// Warns when the per-event ceiling is at or above what the producer will
/// send. Above the cap the ceiling stops being a guard: capture reads the
/// body, builds the event, and the producer refuses it anyway, so the only
/// thing the higher ceiling buys is a later failure. Both sides come from
/// config, so the check stays correct when either knob moves.
fn ai_ceiling_exceeds_producer_cap(config: &Config, producer_message_max_bytes: u32) -> bool {
    let ceiling = config.ai_max_event_bytes;
    // `0` disables the ceiling, so there is no ordering to be wrong about.
    ceiling != 0 && ceiling >= producer_message_max_bytes as u64
}

fn warn_if_ai_ceiling_exceeds_producer_cap(config: &Config, producer_message_max_bytes: u32) {
    if ai_ceiling_exceeds_producer_cap(config, producer_message_max_bytes) {
        warn!(
            ai_max_event_bytes = config.ai_max_event_bytes,
            producer_message_max_bytes,
            "AI_MAX_EVENT_BYTES is at or above KAFKA_INGESTION_PRODUCER_MESSAGE_MAX_BYTES; \
             events between the producer cap and the ceiling are built and then \
             refused by the producer"
        );
    }
}

/// Warns when a token sending full-size AI events would be limited on nearly
/// every one of them, because the window budget cannot fit even a single event
/// at the deployment's ceiling. Both sides come from config, so the check stays
/// correct when either knob moves.
fn warn_if_ai_byte_budget_below_max_event(config: &Config) {
    let max_event_bytes = config.ai_max_event_bytes;
    if max_event_bytes == 0 {
        return;
    }
    let (window_secs, _) = ai_byte_limit_window(config);
    let window_budget = ai_byte_limit_per_second(config).saturating_mul(window_secs);
    if window_budget < max_event_bytes {
        warn!(
            ai_byte_limit_per_second = config.ai_byte_limit_per_second,
            window_secs,
            window_budget,
            max_event_bytes,
            "AI_BYTE_LIMIT_PER_SECOND yields a window budget below AI_MAX_EVENT_BYTES; \
             a token sending full-size events will be limited on nearly every event"
        );
    }
}

async fn create_output_registry(
    config: &Config,
    producer_configs: &HashMap<ProducerName, ProducerConfig>,
    sink_handle: Option<lifecycle::Handle>,
    advisory_handle: Option<lifecycle::Handle>,
) -> anyhow::Result<(OutputRegistry, Option<ProducerRegistry>)> {
    let (output, producers) =
        create_output(config, producer_configs, sink_handle, advisory_handle).await?;
    Ok((OutputRegistry::new(output), producers))
}

async fn create_output(
    config: &Config,
    producer_configs: &HashMap<ProducerName, ProducerConfig>,
    sink_handle: Option<lifecycle::Handle>,
    advisory_handle: Option<lifecycle::Handle>,
) -> anyhow::Result<(Output, Option<ProducerRegistry>)> {
    if config.print_sink {
        return Ok((Output::single(PrintSink {}), None));
    }
    if config.noop_sink {
        info!("NoOpSink enabled, events will be silently dropped");
        return Ok((Output::single(NoOpSink::new()), None));
    }

    // Runs before any producer connects, so a blank topic refuses boot
    // immediately instead of after a broker connect attempt.
    let outputs = OutputTable::new(&config.outputs, config.capture_mode.has_ai_overflow_lane());
    if config.outputs_completeness_check_enabled {
        outputs.check_complete()?;
    } else {
        info!("outputs completeness check disabled; a blank output topic will fail at first produce instead of at boot");
    }

    // With the S3 fallback, the Kafka producer reports to the advisory handle
    // so an unhealthy primary never gates the pod.
    let kafka_handle = if config.s3_fallback_enabled {
        Some(advisory_handle.expect("kafka advisory handle required for fallback"))
    } else {
        sink_handle.clone()
    };
    let producers = ProducerRegistry::build(
        producer_configs,
        kafka_handle
            .clone()
            .map(|handle| (ProducerName::Ingestion, handle))
            .into_iter()
            .collect(),
    )
    .context("failed to start Kafka producers")?;
    let kafka_sink = KafkaSink::new(
        outputs.map_producers(|name| producers.get(*name)),
        config.replay_envelope_compression,
    );

    if !config.s3_fallback_enabled {
        return Ok((Output::single(kafka_sink), Some(producers)));
    }

    let s3_handle = sink_handle.expect("sink lifecycle handle required for S3 fallback");
    let s3_sink = S3Sink::new(
        config
            .s3_fallback_bucket
            .clone()
            .expect("S3 bucket required when fallback enabled"),
        config.s3_fallback_prefix.clone(),
        config.s3_fallback_endpoint.clone(),
        s3_handle,
    )
    .await
    .expect("failed to create S3 sink");

    let output = Output::failover(
        Output::single(kafka_sink),
        Output::single(s3_sink),
        kafka_handle,
    );
    Ok((output, Some(producers)))
}

// Fixed fire-and-forget tuning for the warnings producer. These are
// deliberately not env-configurable: they define the "a warning is worth less
// than the cost of retrying it" contract, not knobs an operator should turn.
// (Queue size in MiB and the per-message byte ceiling stay env-configurable in
// `Config` because they're capacity/safety limits, not the delivery policy.)
const WARNINGS_KAFKA_CLIENT_ID: &str = "capture-ingestion-warnings";
// rdkafka "acks": leader ack only, trading durability for latency.
const WARNINGS_KAFKA_ACKS: &str = "1";
// rdkafka "retries": never retry a dropped warning.
const WARNINGS_KAFKA_RETRIES: u32 = 0;
// Small batching window, in ms.
const WARNINGS_KAFKA_LINGER_MS: u32 = 100;
// Bound the in-flight local backlog by message count.
const WARNINGS_KAFKA_QUEUE_MESSAGES: u32 = 10_000;
// Drop a message not delivered within this many ms.
const WARNINGS_KAFKA_MESSAGE_TIMEOUT_MS: u32 = 5_000;
// Per-type budget per pod: a burst of 60 that refills one a second, so about
// 60 warnings of one type a minute. It caps what a failure that hits many
// tokens at once can send, because the per-token throttle alone scales with
// the number of tokens.
const WARNINGS_TYPE_BUDGET_PERIOD: Duration = Duration::from_secs(1);
const WARNINGS_TYPE_BUDGET_BURST: NonZeroU32 = NonZeroU32::new(60).unwrap();

/// Build the dedicated, warnings-only Kafka config. `hosts`/`tls` come from
/// `CAPTURE_INGESTION_WARNINGS_KAFKA_{HOSTS,TLS}`; everything else is the
/// fixed fire-and-forget policy (client id, acks,
/// retries, linger, queue depth, message timeout) plus the two tunable limits
/// (`queue_mib`, `message_max_bytes`). `..Default::default()` supplies the
/// shared-crate defaults (compression codec, empty client rack) and — critically
/// — leaves every opt-in producer tuning knob at `None`, so warnings never
/// inherit the main event producer's batching/idempotence tuning.
fn build_warnings_kafka_config(
    kafka_hosts: String,
    kafka_tls: bool,
    queue_mib: u32,
    message_max_bytes: u32,
) -> WarningsKafkaConfig {
    WarningsKafkaConfig {
        kafka_hosts,
        kafka_tls,
        kafka_client_id: WARNINGS_KAFKA_CLIENT_ID.to_string(),
        kafka_producer_acks: Some(WARNINGS_KAFKA_ACKS.to_string()),
        kafka_producer_retries: Some(WARNINGS_KAFKA_RETRIES),
        kafka_producer_linger_ms: WARNINGS_KAFKA_LINGER_MS,
        kafka_producer_queue_messages: WARNINGS_KAFKA_QUEUE_MESSAGES,
        kafka_message_timeout_ms: WARNINGS_KAFKA_MESSAGE_TIMEOUT_MS,
        kafka_producer_queue_mib: queue_mib,
        kafka_producer_message_max_bytes: Some(message_max_bytes),
        ..Default::default()
    }
}

/// Build the optional v2 ingestion warnings emitter. Best-effort by contract:
/// any misconfiguration or producer-creation failure logs and returns `None`
/// (capture runs without warnings) instead of failing startup. The producer
/// is a `common_kafka` `ThreadedProducer`, built via
/// `common_kafka::kafka_producer::create_threaded_kafka_producer_no_ping` from a
/// dedicated, warnings-only `common_kafka::config::KafkaConfig` (fire-and-forget
/// acks/retries, a small queue) with `observe_delivery` as its delivery
/// callback. Its destination (hosts/TLS/topic) comes entirely from
/// `CAPTURE_INGESTION_WARNINGS_KAFKA_{HOSTS,TLS,TOPIC}`, never from capture's
/// main event config. When built, a
/// background task heartbeats the advisory lifecycle handle, sweeps the
/// throttle's per-key state, and flushes the producer once at shutdown.
///
/// Uses the no-ping constructor so an unreachable warnings cluster costs
/// capture nothing at boot: no 15s metadata fetch on the startup path, and no
/// pod that serves events for hours with warnings permanently off because the
/// cluster happened to be down the moment it started. librdkafka reconnects on
/// its own, so read `delivered`/`delivery_failed` to judge whether warnings are
/// landing — the enabled gauge only reports that the emitter exists.
async fn create_ingestion_warning_emitter(
    config: &Config,
    handle: Option<lifecycle::Handle>,
) -> Option<Arc<dyn WarningEmitter>> {
    if !config.capture_ingestion_warnings_enabled {
        return None;
    }

    // Past this point the operator asked for warnings, so every exit reports
    // through the gauge with the reason it bailed. Leaving it unset above keeps
    // "disabled" distinct from "enabled but broken".
    let report_disabled = |reason: &'static str| {
        gauge!(INGESTION_WARNINGS_EMITTER_ENABLED, "reason" => reason).set(0.0)
    };

    let hosts = config.capture_ingestion_warnings_kafka_hosts.clone();
    let topic = config.capture_ingestion_warnings_kafka_topic.clone();
    let tls = config.capture_ingestion_warnings_kafka_tls;

    if hosts.is_empty() {
        warn!(
            "ingestion warnings enabled but CAPTURE_INGESTION_WARNINGS_KAFKA_HOSTS is unset; \
             emitter disabled"
        );
        report_disabled("hosts_unset");
        return None;
    }

    if topic.is_empty() {
        warn!(
            "ingestion warnings enabled but CAPTURE_INGESTION_WARNINGS_KAFKA_TOPIC is unset; \
             emitter disabled"
        );
        report_disabled("topic_unset");
        return None;
    }

    let Some(handle) = handle else {
        warn!(
            "ingestion warnings enabled but no lifecycle handle was registered; emitter disabled"
        );
        report_disabled("no_handle");
        return None;
    };

    let warnings_kafka_config = build_warnings_kafka_config(
        hosts,
        tls,
        config.capture_ingestion_warnings_kafka_queue_mib,
        config.capture_ingestion_warnings_kafka_message_max_bytes,
    );

    let producer = match create_threaded_kafka_producer_no_ping(
        &warnings_kafka_config,
        handle.clone(),
        observe_delivery,
    ) {
        Ok(producer) => producer,
        Err(e) => {
            tracing::error!(
                "failed to create ingestion warnings producer, emitter disabled: {e:#}"
            );
            report_disabled("producer_create_failed");
            return None;
        }
    };

    let throttle = WarningThrottle::default()
        .with_type_budget(WARNINGS_TYPE_BUDGET_PERIOD, WARNINGS_TYPE_BUDGET_BURST);
    let emitter = Arc::new(KafkaWarningEmitter::with_throttle(
        producer,
        topic.clone(),
        throttle,
    ));

    let emitter_bg = emitter.clone();
    tokio::spawn(async move {
        let mut heartbeat = tokio::time::interval(Duration::from_secs(10));
        let mut sweep = tokio::time::interval(Duration::from_secs(60));
        loop {
            tokio::select! {
                _ = heartbeat.tick() => handle.report_healthy(),
                _ = sweep.tick() => emitter_bg.sweep_throttle(),
                _ = handle.shutdown_recv() => break,
            }
        }
        // Advisory flush: rdkafka flush blocks, so keep it off the async
        // workers. Dropping the handle after shutdown signals completion.
        let flush_result = tokio::task::spawn_blocking(move || {
            emitter_bg.flush(Duration::from_secs(2));
        })
        .await;
        if let Err(e) = flush_result {
            warn!("ingestion warnings flush task panicked: {e}");
        }
    });

    gauge!(INGESTION_WARNINGS_EMITTER_ENABLED, "reason" => "ok").set(1.0);
    info!(topic = topic.as_str(), "ingestion warnings emitter enabled");
    Some(emitter)
}

fn create_event_restriction_service(
    config: &Config,
    handle: lifecycle::Handle,
    pipelines: Vec<Pipeline>,
) -> Option<EventRestrictionService> {
    if !config.event_restrictions_enabled {
        return None;
    }

    let Some(ref redis_url) = config.event_restrictions_redis_url else {
        warn!("Event restrictions enabled but EVENT_RESTRICTIONS_REDIS_URL not set");
        return None;
    };

    let pipelines_for_log = pipelines.clone();
    let service = EventRestrictionService::new(
        pipelines,
        Duration::from_secs(config.event_restrictions_fail_open_after_secs),
    );

    let service_clone = service.clone();
    let refresh_interval = Duration::from_secs(config.event_restrictions_refresh_interval_secs);

    let redis_url = redis_url.clone();
    let response_timeout = if config.redis_response_timeout_ms == 0 {
        None
    } else {
        Some(Duration::from_millis(config.redis_response_timeout_ms))
    };
    let connection_timeout = if config.redis_connection_timeout_ms == 0 {
        None
    } else {
        Some(Duration::from_millis(config.redis_connection_timeout_ms))
    };

    tokio::spawn(async move {
        service_clone
            .start_refresh_task(
                || {
                    let url = redis_url.clone();
                    async move {
                        let repo = RedisRestrictionsRepository::new(
                            url,
                            response_timeout,
                            connection_timeout,
                        )
                        .await?;
                        let result: Arc<
                            dyn crate::event_restrictions::EventRestrictionsRepository,
                        > = Arc::new(repo);
                        Ok(result)
                    }
                },
                refresh_interval,
                handle,
            )
            .await;
    });

    info!(
        pipelines = ?pipelines_for_log,
        refresh_interval_secs = config.event_restrictions_refresh_interval_secs,
        fail_open_after_secs = config.event_restrictions_fail_open_after_secs,
        "Event restrictions enabled"
    );

    Some(service)
}

#[cfg(test)]
mod tests {
    use super::*;
    use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
    use rstest::rstest;
    use std::collections::HashMap;

    fn warnings_config(enabled: bool, hosts: &str, topic: &str) -> Config {
        let cfg_env: HashMap<String, String> = [
            ("REDIS_URL", "redis://localhost:6379/"),
            ("CAPTURE_MODE", "events"),
            (
                "CAPTURE_OUTPUT_ANALYTICS_MAIN_TOPIC",
                "events_plugin_ingestion",
            ),
            (
                "CAPTURE_INGESTION_WARNINGS_ENABLED",
                if enabled { "true" } else { "false" },
            ),
            ("CAPTURE_INGESTION_WARNINGS_KAFKA_HOSTS", hosts),
            ("CAPTURE_INGESTION_WARNINGS_KAFKA_TOPIC", topic),
        ]
        .into_iter()
        .map(|(k, v)| (k.to_string(), v.to_string()))
        .collect();
        envconfig::Envconfig::init_from_hashmap(&cfg_env).expect("test config")
    }

    /// The emitter gauge as `(reason, value)`, or `None` if never emitted.
    fn emitter_enabled_gauge(snapshotter: &Snapshotter) -> Option<(String, f64)> {
        snapshotter
            .snapshot()
            .into_vec()
            .into_iter()
            .find(|(ckey, _, _, _)| ckey.key().name() == INGESTION_WARNINGS_EMITTER_ENABLED)
            .map(|(ckey, _, _, value)| {
                let reason = ckey
                    .key()
                    .labels()
                    .find(|label| label.key() == "reason")
                    .map(|label| label.value().to_string())
                    .expect("every emission must carry a reason label");
                match value {
                    DebugValue::Gauge(v) => (reason, v.into_inner()),
                    other => panic!("expected a gauge, got {other:?}"),
                }
            })
    }

    /// Run `f` with metrics captured locally. The recorder is thread-scoped, so
    /// callers drive futures on this thread via `current_thread_runtime`.
    fn capture_metrics<T>(f: impl FnOnce() -> T) -> (T, Snapshotter) {
        let recorder = DebuggingRecorder::new();
        let snapshotter = recorder.snapshotter();
        let out = metrics::with_local_recorder(&recorder, f);
        (out, snapshotter)
    }

    fn current_thread_runtime() -> tokio::runtime::Runtime {
        tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()
            .expect("runtime")
    }

    struct AiLimiterInput {
        capture_mode: &'static str,
        enabled: Option<&'static str>,
        overflow_topic: Option<&'static str>,
    }

    fn ai_limiter_config(input: &AiLimiterInput) -> Config {
        let mut cfg_env: HashMap<String, String> = [
            ("REDIS_URL", "redis://localhost:6379/"),
            ("CAPTURE_MODE", input.capture_mode),
            (
                "CAPTURE_OUTPUT_ANALYTICS_MAIN_TOPIC",
                "events_plugin_ingestion",
            ),
        ]
        .into_iter()
        .map(|(k, v)| (k.to_string(), v.to_string()))
        .collect();
        if let Some(enabled) = input.enabled {
            cfg_env.insert(
                "AI_EVENTS_OVERFLOW_ENABLED".to_string(),
                enabled.to_string(),
            );
        }
        if let Some(topic) = input.overflow_topic {
            cfg_env.insert(
                "CAPTURE_OUTPUT_AI_OVERFLOW_TOPIC".to_string(),
                topic.to_string(),
            );
        }
        envconfig::Envconfig::init_from_hashmap(&cfg_env).expect("test config")
    }

    #[rstest]
    #[case::events_enabled(
        AiLimiterInput {
            capture_mode: "events",
            enabled: Some("true"),
            overflow_topic: None,
        },
        true
    )]
    #[case::events_topic_without_flag(
        AiLimiterInput {
            capture_mode: "events",
            enabled: None,
            overflow_topic: Some("events_plugin_ingestion_ai_overflow"),
        },
        false
    )]
    #[case::import_disabled(
        AiLimiterInput {
            capture_mode: "import",
            enabled: Some("false"),
            overflow_topic: Some("events_plugin_ingestion_ai_overflow"),
        },
        false
    )]
    fn ai_events_overflow_limiter_follows_only_the_flag(
        #[case] input: AiLimiterInput,
        #[case] expected_enabled: bool,
    ) {
        assert_eq!(
            ai_events_overflow_limiter_enabled(&ai_limiter_config(&input)),
            expected_enabled
        );
    }

    /// Signature shared by the limiter constructors under test.
    type LimiterBuilder = fn(
        &Config,
        Vec<Arc<dyn common_redis::Client + Send + Sync>>,
    ) -> anyhow::Result<GlobalRateLimiter>;

    /// A zero window gives every bucket an infinite leak rate, so the limiter
    /// admits everything. Building one must fail at boot rather than run as a
    /// limiter that never limits.
    #[rstest]
    #[case::ai_bytes(GlobalRateLimiter::new_ai_bytes)]
    #[case::token_distinct_id(GlobalRateLimiter::new_token_distinct_id)]
    fn limiters_reject_a_zero_window(#[case] build: LimiterBuilder) {
        let cfg_env: HashMap<String, String> = [
            ("REDIS_URL", "redis://localhost:6379/"),
            ("CAPTURE_MODE", "events"),
            (
                "CAPTURE_OUTPUT_ANALYTICS_MAIN_TOPIC",
                "events_plugin_ingestion",
            ),
            ("GLOBAL_RATE_LIMIT_WINDOW_INTERVAL_SECS", "0"),
        ]
        .into_iter()
        .map(|(k, v)| (k.to_string(), v.to_string()))
        .collect();
        let config: Config =
            envconfig::Envconfig::init_from_hashmap(&cfg_env).expect("test config");

        // `GlobalRateLimiter` is not `Debug`, so unwrap the error by hand.
        let err = match build(&config, vec![]) {
            Ok(_) => panic!("a zero window must not build a limiter"),
            Err(err) => err,
        };
        assert!(
            err.to_string()
                .contains("GLOBAL_RATE_LIMIT_WINDOW_INTERVAL_SECS"),
            "the error must name the offending setting, got: {err}"
        );
    }

    /// The AI byte limiter takes its window from its own knob when one is set,
    /// so a zero there has to be caught and named separately. Naming the wrong
    /// variable sends an operator to a setting that is already correct.
    #[test]
    fn ai_byte_limiter_rejects_a_zero_window_from_its_own_knob() {
        let cfg_env: HashMap<String, String> = [
            ("REDIS_URL", "redis://localhost:6379/"),
            ("CAPTURE_MODE", "events"),
            (
                "CAPTURE_OUTPUT_ANALYTICS_MAIN_TOPIC",
                "events_plugin_ingestion",
            ),
            ("GLOBAL_RATE_LIMIT_WINDOW_INTERVAL_SECS", "180"),
            ("AI_BYTE_LIMIT_WINDOW_INTERVAL_SECS", "0"),
        ]
        .into_iter()
        .map(|(k, v)| (k.to_string(), v.to_string()))
        .collect();
        let config: Config =
            envconfig::Envconfig::init_from_hashmap(&cfg_env).expect("test config");

        let err = match GlobalRateLimiter::new_ai_bytes(&config, vec![]) {
            Ok(_) => panic!("a zero AI byte window must not build a limiter"),
            Err(err) => err,
        };
        assert!(
            err.to_string()
                .contains("AI_BYTE_LIMIT_WINDOW_INTERVAL_SECS"),
            "the error must name the AI byte knob, not the shared one, got: {err}"
        );

        let err = match GlobalRateLimiter::new_token_distinct_id(&config, vec![]) {
            Ok(_) => panic!("an empty redis instance list must not build a limiter"),
            Err(err) => err,
        };
        assert!(
            !err.to_string().contains("WINDOW_INTERVAL_SECS"),
            "the token+distinct_id limiter must not fail on a window, got: {err}"
        );
    }

    #[test]
    #[should_panic(expected = "imports must never overflow")]
    fn ai_events_overflow_limiter_is_refused_in_import_mode() {
        ai_events_overflow_limiter_enabled(&ai_limiter_config(&AiLimiterInput {
            capture_mode: "import",
            enabled: Some("true"),
            overflow_topic: None,
        }));
    }

    #[test]
    fn warnings_kafka_config_is_isolated_from_main_producer_tuning() {
        let cfg = build_warnings_kafka_config("broker:9092".to_string(), true, 16, 1_048_576);

        assert_eq!(cfg.kafka_hosts, "broker:9092");
        assert!(cfg.kafka_tls);

        // Fixed fire-and-forget policy (not env-configurable).
        assert_eq!(cfg.kafka_client_id, WARNINGS_KAFKA_CLIENT_ID);
        assert_eq!(
            cfg.kafka_producer_acks.as_deref(),
            Some(WARNINGS_KAFKA_ACKS)
        );
        assert_eq!(cfg.kafka_producer_retries, Some(WARNINGS_KAFKA_RETRIES));
        assert_eq!(cfg.kafka_producer_linger_ms, WARNINGS_KAFKA_LINGER_MS);
        assert_eq!(
            cfg.kafka_producer_queue_messages,
            WARNINGS_KAFKA_QUEUE_MESSAGES
        );
        assert_eq!(
            cfg.kafka_message_timeout_ms,
            WARNINGS_KAFKA_MESSAGE_TIMEOUT_MS
        );

        // The two tunable capacity/safety limits pass through unchanged.
        assert_eq!(cfg.kafka_producer_queue_mib, 16);
        assert_eq!(cfg.kafka_producer_message_max_bytes, Some(1_048_576));

        // Critically: none of the main event producer's opt-in tuning leaks in
        // — `..Default::default()` must leave every WarpStream knob at `None`.
        assert_eq!(cfg.kafka_producer_batch_size, None);
        assert_eq!(cfg.kafka_producer_batch_num_messages, None);
        assert_eq!(cfg.kafka_producer_enable_idempotence, None);
        assert_eq!(
            cfg.kafka_producer_max_in_flight_requests_per_connection,
            None
        );
        assert_eq!(cfg.kafka_producer_topic_metadata_refresh_interval_ms, None);
        assert_eq!(cfg.kafka_producer_sticky_partitioning_linger_ms, None);
        assert_eq!(cfg.kafka_producer_partitioner, None);
    }

    #[tokio::test]
    async fn ingestion_warnings_emitter_does_not_consult_v0_kafka_block() {
        // Regression guard for the v0 fallback removal: with warnings enabled but
        // no dedicated hosts, the emitter must stay disabled rather than reuse the
        // v0 KAFKA_HOSTS / CAPTURE_OUTPUT_CLIENT_WARNINGS_TOPIC block. If the
        // fallback were still live, it would build a producer against v0-broker.
        let cfg_env: HashMap<String, String> = [
            ("REDIS_URL", "redis://localhost:6379/"),
            ("CAPTURE_MODE", "events"),
            (
                "CAPTURE_OUTPUT_ANALYTICS_MAIN_TOPIC",
                "events_plugin_ingestion",
            ),
            ("CAPTURE_OUTPUT_CLIENT_WARNINGS_TOPIC", "v0-warnings-topic"),
            ("CAPTURE_INGESTION_WARNINGS_ENABLED", "true"),
            // CAPTURE_INGESTION_WARNINGS_KAFKA_HOSTS deliberately left unset.
        ]
        .into_iter()
        .map(|(k, v)| (k.to_string(), v.to_string()))
        .collect();
        let config: Config =
            envconfig::Envconfig::init_from_hashmap(&cfg_env).expect("test config");

        assert!(config.capture_ingestion_warnings_kafka_hosts.is_empty());

        let emitter = create_ingestion_warning_emitter(&config, None).await;
        assert!(
            emitter.is_none(),
            "emitter must stay disabled on empty dedicated hosts, not fall back to KAFKA_HOSTS"
        );
    }

    #[tokio::test]
    async fn disabled_emitter_with_live_handle_does_not_shut_capture_down() {
        // `register_components` moves the warnings handle in here and keeps no
        // clone, so bailing out early drops the last reference and fires
        // `ComponentEvent::Died`. Removing the v0 fallback is what makes that
        // reachable in production: a pod told to emit warnings but given no
        // dedicated hosts now takes this path on every start. Warnings are
        // best-effort, so it has to cost capture nothing.
        let cfg_env: HashMap<String, String> = [
            ("REDIS_URL", "redis://localhost:6379/"),
            ("CAPTURE_MODE", "events"),
            (
                "CAPTURE_OUTPUT_ANALYTICS_MAIN_TOPIC",
                "events_plugin_ingestion",
            ),
            ("CAPTURE_INGESTION_WARNINGS_ENABLED", "true"),
        ]
        .into_iter()
        .map(|(k, v)| (k.to_string(), v.to_string()))
        .collect();
        let config: Config =
            envconfig::Envconfig::init_from_hashmap(&cfg_env).expect("test config");

        let shutdown = tokio_util::sync::CancellationToken::new();
        let mut manager = lifecycle::Manager::builder("test")
            .with_trap_signals(false)
            .with_prestop_check(false)
            .with_shutdown_token(shutdown.clone())
            .build();
        // Mirrors the registration in `register_components`.
        let handle = manager.register(
            "ingestion-warnings",
            lifecycle::ComponentOptions::new()
                .with_liveness_deadline(Duration::from_secs(30))
                .is_advisory(true),
        );
        let server = manager.register("server", lifecycle::ComponentOptions::new());
        let _guard = manager.monitor_background();

        let emitter = create_ingestion_warning_emitter(&config, Some(handle)).await;
        assert!(emitter.is_none(), "emitter must stay disabled");

        // Long enough for a Died-driven cancel to have landed.
        tokio::time::sleep(Duration::from_millis(300)).await;
        assert!(
            !shutdown.is_cancelled(),
            "a disabled warnings emitter must not initiate capture shutdown"
        );
        assert!(!server.is_shutting_down(), "capture must still be serving");
    }

    #[rstest::rstest]
    #[case(CaptureMode::Events)]
    #[case(CaptureMode::Recordings)]
    #[case(CaptureMode::Ai)]
    #[case(CaptureMode::Import)]
    #[tokio::test]
    async fn create_sink_refuses_boot_on_missing_output_topic(#[case] mode: CaptureMode) {
        let cfg_env: HashMap<String, String> = [
            ("REDIS_URL", "redis://localhost:6379/"),
            ("CAPTURE_MODE", mode.as_tag()),
        ]
        .into_iter()
        .map(|(k, v)| (k.to_string(), v.to_string()))
        .collect();
        let mut config: Config =
            envconfig::Envconfig::init_from_hashmap(&cfg_env).expect("test config");
        config.outputs_completeness_check_enabled = true;
        config.outputs.dlq_topic = String::new();
        let default_producers = producers::load_all(&HashMap::new()).expect("default producers");

        let err = create_output_registry(&config, &default_producers, None, None)
            .await
            .err()
            .expect("boot must be refused when an output topic is empty");
        let msg = format!("{err:#}");
        assert!(
            msg.contains("dlq"),
            "error should name the missing output: {msg}"
        );

        // The default: with the check off, the same blank topic boots (and
        // would fail at first produce instead).
        config.outputs_completeness_check_enabled = false;
        create_output_registry(&config, &default_producers, None, None)
            .await
            .expect("boot must proceed when the completeness check is disabled");
    }

    /// The ceiling only guards anything while it sits under the producer's cap.
    /// A deployment that raises the producer keeps its headroom; one that never
    /// touched it gets told the default is too high for its broker.
    #[rstest::rstest]
    #[case::default_ceiling_on_a_default_producer(8_388_608, 1_000_000, true)]
    #[case::default_ceiling_under_a_raised_producer(8_388_608, 10_485_760, false)]
    #[case::equal_still_warns(1_000_000, 1_000_000, true)]
    #[case::disabled_ceiling_never_warns(0, 1_000_000, false)]
    fn ai_ceiling_is_checked_against_the_producer_cap(
        #[case] ceiling: u64,
        #[case] producer_cap: u32,
        #[case] expected: bool,
    ) {
        let cfg_env: HashMap<String, String> = [
            ("REDIS_URL", "redis://localhost:6379/"),
            ("CAPTURE_MODE", "ai"),
            (
                "CAPTURE_OUTPUT_ANALYTICS_MAIN_TOPIC",
                "events_plugin_ingestion_ai",
            ),
        ]
        .into_iter()
        .map(|(k, v)| (k.to_string(), v.to_string()))
        .collect();
        let mut config: Config =
            envconfig::Envconfig::init_from_hashmap(&cfg_env).expect("test config");
        config.ai_max_event_bytes = ceiling;

        assert_eq!(
            ai_ceiling_exceeds_producer_cap(&config, producer_cap),
            expected
        );
    }

    /// Import deployments never build the AI byte limiter, however the knob is
    /// set — that omission is the entire import exemption, so a rate leaking
    /// through here would start throttling backfills.
    #[rstest::rstest]
    #[case::events_keeps_the_configured_rate(CaptureMode::Events, 5_000, 5_000)]
    #[case::ai_keeps_the_configured_rate(CaptureMode::Ai, 5_000, 5_000)]
    #[case::import_is_exempt(CaptureMode::Import, 5_000, 0)]
    #[case::unset_stays_unset(CaptureMode::Events, 0, 0)]
    fn ai_byte_limit_per_second_by_mode(
        #[case] mode: CaptureMode,
        #[case] configured: u64,
        #[case] expected: u64,
    ) {
        let cfg_env: HashMap<String, String> = [
            ("REDIS_URL", "redis://localhost:6379/"),
            ("CAPTURE_MODE", mode.as_tag()),
        ]
        .into_iter()
        .map(|(k, v)| (k.to_string(), v.to_string()))
        .collect();
        let mut config: Config =
            envconfig::Envconfig::init_from_hashmap(&cfg_env).expect("test config");
        config.ai_byte_limit_per_second = configured;

        assert_eq!(ai_byte_limit_per_second(&config), expected);
    }

    /// Absent gauge means warnings are off on purpose; `0` means an operator
    /// asked for them and we could not build them. `reason` says which.
    #[rstest]
    #[case::off_on_purpose(false, "", "", None)]
    #[case::hosts_unset(true, "", "", Some("hosts_unset"))]
    #[case::topic_unset(true, "broker:9092", "", Some("topic_unset"))]
    #[case::no_handle(true, "broker:9092", "client_ingestion_warning", Some("no_handle"))]
    fn emitter_gauge_reports_why_warnings_are_off(
        #[case] enabled: bool,
        #[case] hosts: &str,
        #[case] topic: &str,
        #[case] expected_reason: Option<&str>,
    ) {
        let config = warnings_config(enabled, hosts, topic);
        let runtime = current_thread_runtime();

        let (emitter, snapshotter) =
            capture_metrics(|| runtime.block_on(create_ingestion_warning_emitter(&config, None)));

        assert!(emitter.is_none(), "no case here can build an emitter");
        match expected_reason {
            Some(reason) => assert_eq!(
                emitter_enabled_gauge(&snapshotter),
                Some((reason.to_string(), 0.0)),
            ),
            None => assert!(
                emitter_enabled_gauge(&snapshotter).is_none(),
                "disabled must stay unset, or it looks misconfigured",
            ),
        }
    }

    #[test]
    fn healthy_emitter_reports_ok() {
        // TEST-NET-1 host: the no-ping constructor never dials it, so the real
        // success path runs offline.
        let config = warnings_config(true, "192.0.2.1:9092", "client_ingestion_warning");
        let mut manager = lifecycle::Manager::builder("test")
            .with_trap_signals(false)
            .with_prestop_check(false)
            .build();
        // Mirrors the registration in `register_components`.
        let handle = manager.register(
            "ingestion-warnings",
            lifecycle::ComponentOptions::new()
                .with_liveness_deadline(Duration::from_secs(30))
                .is_advisory(true),
        );
        let runtime = current_thread_runtime();

        let (emitter, snapshotter) = capture_metrics(|| {
            runtime.block_on(create_ingestion_warning_emitter(&config, Some(handle)))
        });

        assert!(emitter.is_some(), "emitter must be built");
        assert_eq!(
            emitter_enabled_gauge(&snapshotter),
            Some(("ok".to_string(), 1.0)),
            "healthy and failed emissions must share a label set",
        );
    }
}