Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

16.5 KB
Raw Blame
//! Binds every fixed routing [`Destination`] to its output: a Kafka topic and
//! the named producer that carries it.

use std::sync::Arc;

use crate::config::OutputsConfig;
use crate::pipeline::{Address, AiLane, AnalyticsLane, BasicLane, PipelineLane, SessionReplayLane};
use crate::producers::ProducerName;

/// Which configured output a routing decision selects, named by pipeline and
/// lane. Distinct outputs may share a topic.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Destination {
    AnalyticsMain,
    AnalyticsOverflow,
    AnalyticsHistorical,
    ClientWarningsMain,
    HeatmapsMain,
    SessionReplayMain,
    SessionReplayOverflow,
    /// Every pipeline's dlq lane, one shared output.
    Dlq,
    ErrorTrackingMain,
    AiMain,
    AiOverflow,
    /// Admin-configured topic copied from `redirect_to_topic`.
    Custom(String),
}

impl Destination {
    /// Every registered always-required output. `check_complete` walks this so
    /// a newly added output is caught at boot rather than at first produce.
    const REGISTERED: [Destination; 11] = [
        Destination::AnalyticsMain,
        Destination::AnalyticsOverflow,
        Destination::AnalyticsHistorical,
        Destination::ClientWarningsMain,
        Destination::HeatmapsMain,
        Destination::SessionReplayMain,
        Destination::SessionReplayOverflow,
        Destination::Dlq,
        Destination::ErrorTrackingMain,
        Destination::AiMain,
        Destination::AiOverflow,
    ];

    pub(crate) fn for_lane(lane: PipelineLane) -> Destination {
        match lane {
            PipelineLane::Analytics(AnalyticsLane::Main) => Destination::AnalyticsMain,
            PipelineLane::Analytics(AnalyticsLane::Overflow) => Destination::AnalyticsOverflow,
            PipelineLane::Analytics(AnalyticsLane::Historical) => Destination::AnalyticsHistorical,
            PipelineLane::Ai(AiLane::Main) => Destination::AiMain,
            PipelineLane::Ai(AiLane::Overflow) => Destination::AiOverflow,
            PipelineLane::Warnings(BasicLane::Main) => Destination::ClientWarningsMain,
            PipelineLane::Heatmaps(BasicLane::Main) => Destination::HeatmapsMain,
            PipelineLane::ErrorTracking(BasicLane::Main) => Destination::ErrorTrackingMain,
            PipelineLane::Replay(SessionReplayLane::Main) => Destination::SessionReplayMain,
            PipelineLane::Replay(SessionReplayLane::Overflow) => Destination::SessionReplayOverflow,
        }
    }

    pub(crate) fn for_address(address: Address) -> Destination {
        match address {
            Address::Lane(lane) => Self::for_lane(lane),
            Address::Dlq => Destination::Dlq,
            Address::Custom(topic) => Destination::Custom(topic),
        }
    }

    /// Whether this output participates in the boot completeness check.
    /// Deliberately exhaustive: a new variant cannot compile without
    /// declaring which side it is on, and
    /// `registered_is_exactly_the_required_outputs` pins [`Self::REGISTERED`]
    /// to the `true` arms so declaring `true` without joining the array
    /// fails a test instead of silently skipping the check.
    #[cfg(test)]
    fn is_required(&self) -> bool {
        match self {
            Destination::AnalyticsMain
            | Destination::AnalyticsOverflow
            | Destination::AnalyticsHistorical
            | Destination::ClientWarningsMain
            | Destination::HeatmapsMain
            | Destination::SessionReplayMain
            | Destination::SessionReplayOverflow
            | Destination::Dlq
            | Destination::ErrorTrackingMain
            | Destination::AiMain
            | Destination::AiOverflow => true,
            // A custom redirect carries its own topic per event.
            Destination::Custom(_) => false,
        }
    }

    /// Stable, low-cardinality label for diagnostics. `Custom` collapses to
    /// "custom" so admin topic names never leak into error messages.
    fn name(&self) -> &'static str {
        match self {
            Destination::AnalyticsMain => "analytics-main",
            Destination::AnalyticsOverflow => "analytics-overflow",
            Destination::AnalyticsHistorical => "analytics-historical",
            Destination::ClientWarningsMain => "clientwarnings-main",
            Destination::HeatmapsMain => "heatmaps-main",
            Destination::SessionReplayMain => "sessionreplay-main",
            Destination::SessionReplayOverflow => "sessionreplay-overflow",
            Destination::Dlq => "dlq",
            Destination::ErrorTrackingMain => "errortracking-main",
            Destination::AiMain => "ai-main",
            Destination::AiOverflow => "ai-overflow",
            Destination::Custom(_) => "custom",
        }
    }
}

/// `R` is how the target names its producer: a [`ProducerName`] in config,
/// the producer handle once setup resolves it.
#[derive(Clone, Debug)]
pub struct OutputTarget<R = ProducerName> {
    // `Arc<str>` so the per-record lookup, metric labels and produce record never allocate.
    pub(crate) topic: Arc<str>,
    pub(crate) producer: R,
}

#[derive(Clone, Debug)]
pub struct OutputTable<R = ProducerName> {
    pub(crate) analytics_main: OutputTarget<R>,
    pub(crate) analytics_overflow: OutputTarget<R>,
    pub(crate) analytics_historical: OutputTarget<R>,
    pub(crate) session_replay_main: OutputTarget<R>,
    pub(crate) session_replay_overflow: OutputTarget<R>,
    pub(crate) client_warnings: OutputTarget<R>,
    pub(crate) heatmaps: OutputTarget<R>,
    pub(crate) dlq: OutputTarget<R>,
    pub(crate) error_tracking: OutputTarget<R>,
    pub(crate) ai_main: OutputTarget<R>,
    pub(crate) ai_overflow: OutputTarget<R>,
    pub(crate) has_ai_overflow_lane: bool,
    pub(crate) custom_producer: R,
}

impl<R> OutputTable<R> {
    fn target_for(&self, output: &Destination) -> Option<&OutputTarget<R>> {
        match output {
            Destination::AnalyticsMain => Some(&self.analytics_main),
            Destination::AnalyticsOverflow => Some(&self.analytics_overflow),
            Destination::AnalyticsHistorical => Some(&self.analytics_historical),
            Destination::ClientWarningsMain => Some(&self.client_warnings),
            Destination::HeatmapsMain => Some(&self.heatmaps),
            Destination::SessionReplayMain => Some(&self.session_replay_main),
            Destination::SessionReplayOverflow => Some(&self.session_replay_overflow),
            Destination::Dlq => Some(&self.dlq),
            Destination::ErrorTrackingMain => Some(&self.error_tracking),
            Destination::AiMain => Some(&self.ai_main),
            Destination::AiOverflow => Some(&self.ai_overflow),
            Destination::Custom(_) => None,
        }
    }

    pub fn topic_for<'a>(&'a self, output: &'a Destination) -> &'a str {
        match (output, self.target_for(output)) {
            (Destination::Custom(topic), _) => topic,
            (_, Some(target)) => &target.topic,
            (_, None) => unreachable!("every fixed output has a target"),
        }
    }

    pub(crate) fn resolve(&self, output: &Destination) -> (Arc<str>, &R) {
        match (output, self.target_for(output)) {
            (_, Some(target)) => (Arc::clone(&target.topic), &target.producer),
            (Destination::Custom(topic), None) => {
                (Arc::from(topic.as_str()), &self.custom_producer)
            }
            (_, None) => unreachable!("every fixed output has a target"),
        }
    }

    pub fn map_producers<T>(&self, mut f: impl FnMut(&R) -> T) -> OutputTable<T> {
        let custom_producer = f(&self.custom_producer);
        let mut target = |t: &OutputTarget<R>| OutputTarget {
            topic: Arc::clone(&t.topic),
            producer: f(&t.producer),
        };
        OutputTable {
            analytics_main: target(&self.analytics_main),
            analytics_overflow: target(&self.analytics_overflow),
            analytics_historical: target(&self.analytics_historical),
            session_replay_main: target(&self.session_replay_main),
            session_replay_overflow: target(&self.session_replay_overflow),
            client_warnings: target(&self.client_warnings),
            heatmaps: target(&self.heatmaps),
            dlq: target(&self.dlq),
            error_tracking: target(&self.error_tracking),
            ai_main: target(&self.ai_main),
            ai_overflow: target(&self.ai_overflow),
            has_ai_overflow_lane: self.has_ai_overflow_lane,
            custom_producer,
        }
    }

    pub fn has_ai_overflow_lane(&self) -> bool {
        self.has_ai_overflow_lane
    }

    /// Every registered output must resolve to a non-empty topic. `Custom` is
    /// excluded because it carries its own topic per event.
    pub fn check_complete(&self) -> anyhow::Result<()> {
        for output in &Destination::REGISTERED {
            anyhow::ensure!(
                !self.topic_for(output).is_empty(),
                "output '{}' resolves to an empty Kafka topic; every non-custom \
                 output must be bound to a configured, non-empty topic",
                output.name(),
            );
        }
        Ok(())
    }
}

impl OutputTable {
    pub fn new(config: &OutputsConfig, has_ai_overflow_lane: bool) -> Self {
        let target = |topic: &String, producer: ProducerName| OutputTarget {
            topic: Arc::from(topic.as_str()),
            producer,
        };
        Self {
            analytics_main: target(&config.analytics_main_topic, config.analytics_main_producer),
            analytics_overflow: target(
                &config.analytics_overflow_topic,
                config.analytics_overflow_producer,
            ),
            analytics_historical: target(
                &config.analytics_historical_topic,
                config.analytics_historical_producer,
            ),
            session_replay_main: target(
                &config.session_replay_main_topic,
                config.session_replay_main_producer,
            ),
            session_replay_overflow: target(
                &config.session_replay_overflow_topic,
                config.session_replay_overflow_producer,
            ),
            client_warnings: target(
                &config.client_warnings_topic,
                config.client_warnings_producer,
            ),
            heatmaps: target(&config.heatmaps_topic, config.heatmaps_producer),
            dlq: target(&config.dlq_topic, config.dlq_producer),
            error_tracking: target(&config.error_tracking_topic, config.error_tracking_producer),
            ai_main: target(&config.ai_main_topic, config.ai_main_producer),
            ai_overflow: target(&config.ai_overflow_topic, config.ai_overflow_producer),
            has_ai_overflow_lane,
            custom_producer: config.custom_producer,
        }
    }
}

#[cfg(test)]
pub(crate) fn test_outputs() -> OutputTable {
    let target = |topic: &str| OutputTarget {
        topic: Arc::from(topic),
        producer: ProducerName::Ingestion,
    };
    OutputTable {
        analytics_main: target("events_plugin_ingestion"),
        analytics_overflow: target("events_plugin_ingestion_overflow"),
        analytics_historical: target("events_plugin_ingestion_historical"),
        session_replay_main: target("events_plugin_ingestion"),
        session_replay_overflow: target("replay_overflow"),
        client_warnings: target("client_ingestion_warning"),
        heatmaps: target("heatmaps"),
        dlq: target("events_plugin_ingestion_dlq"),
        error_tracking: target("error_tracking_events"),
        ai_main: target("ai_events"),
        ai_overflow: target("ai_events_overflow"),
        has_ai_overflow_lane: true,
        custom_producer: ProducerName::Ingestion,
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use rstest::rstest;

    #[rstest]
    #[case(Destination::AnalyticsMain, "events_plugin_ingestion")]
    #[case(Destination::SessionReplayMain, "events_plugin_ingestion")]
    #[case(Destination::AnalyticsOverflow, "events_plugin_ingestion_overflow")]
    #[case(Destination::AnalyticsHistorical, "events_plugin_ingestion_historical")]
    #[case(Destination::ClientWarningsMain, "client_ingestion_warning")]
    #[case(Destination::HeatmapsMain, "heatmaps")]
    #[case(Destination::SessionReplayOverflow, "replay_overflow")]
    #[case(Destination::Dlq, "events_plugin_ingestion_dlq")]
    #[case(Destination::ErrorTrackingMain, "error_tracking_events")]
    #[case(Destination::AiMain, "ai_events")]
    #[case(Destination::AiOverflow, "ai_events_overflow")]
    fn topic_for_resolves_registered_outputs(#[case] output: Destination, #[case] expected: &str) {
        assert_eq!(test_outputs().topic_for(&output), expected);
    }

    #[test]
    fn topic_for_custom_returns_inline_topic() {
        let registry = test_outputs();
        assert_eq!(
            registry.topic_for(&Destination::Custom("admin_topic".to_string())),
            "admin_topic"
        );
    }

    #[test]
    fn session_replay_main_does_not_share_analytics_main() {
        let mut registry = test_outputs();
        registry.session_replay_main.topic = Arc::from("replay_main");
        assert_eq!(
            registry.topic_for(&Destination::SessionReplayMain),
            "replay_main"
        );
        assert_eq!(
            registry.topic_for(&Destination::AnalyticsMain),
            "events_plugin_ingestion"
        );
    }

    #[test]
    fn each_output_resolves_to_its_own_producer() {
        let mut next = 0;
        let table = test_outputs().map_producers(|_| {
            next += 1;
            next
        });
        let custom = Destination::Custom("admin_topic".to_string());
        let (topic, _) = table.resolve(&custom);
        assert_eq!(&*topic, "admin_topic");

        let producers: std::collections::HashSet<i32> = Destination::REGISTERED
            .iter()
            .chain([&custom])
            .map(|output| *table.resolve(output).1)
            .collect();
        assert_eq!(producers.len(), Destination::REGISTERED.len() + 1);
    }

    #[test]
    fn check_complete_accepts_full_registry() {
        assert!(test_outputs().check_complete().is_ok());
    }

    /// The all-variants list below is hand-maintained: grow it with the enum.
    #[test]
    fn registered_is_exactly_the_required_outputs() {
        let all = [
            Destination::AnalyticsMain,
            Destination::AnalyticsOverflow,
            Destination::AnalyticsHistorical,
            Destination::ClientWarningsMain,
            Destination::HeatmapsMain,
            Destination::SessionReplayMain,
            Destination::SessionReplayOverflow,
            Destination::Dlq,
            Destination::ErrorTrackingMain,
            Destination::AiMain,
            Destination::AiOverflow,
            Destination::Custom("t".to_string()),
        ];
        for output in &all {
            assert_eq!(
                Destination::REGISTERED.contains(output),
                output.is_required(),
                "'{}' must be in REGISTERED exactly when it is required",
                output.name()
            );
        }
    }

    /// Every registered output, blanked one at a time, must fail the check and
    /// the error must name the offending output.
    #[rstest]
    #[case("analytics-main", |r: &mut OutputTable| r.analytics_main.topic = Arc::from(""))]
    #[case("analytics-overflow", |r: &mut OutputTable| r.analytics_overflow.topic = Arc::from(""))]
    #[case("analytics-historical", |r: &mut OutputTable| r.analytics_historical.topic = Arc::from(""))]
    #[case("clientwarnings-main", |r: &mut OutputTable| r.client_warnings.topic = Arc::from(""))]
    #[case("heatmaps-main", |r: &mut OutputTable| r.heatmaps.topic = Arc::from(""))]
    #[case("sessionreplay-main", |r: &mut OutputTable| r.session_replay_main.topic = Arc::from(""))]
    #[case("sessionreplay-overflow", |r: &mut OutputTable| r.session_replay_overflow.topic = Arc::from(""))]
    #[case("dlq", |r: &mut OutputTable| r.dlq.topic = Arc::from(""))]
    #[case("errortracking-main", |r: &mut OutputTable| r.error_tracking.topic = Arc::from(""))]
    #[case("ai-main", |r: &mut OutputTable| r.ai_main.topic = Arc::from(""))]
    #[case("ai-overflow", |r: &mut OutputTable| r.ai_overflow.topic = Arc::from(""))]
    fn check_complete_rejects_empty_topic(
        #[case] output_name: &str,
        #[case] blank: fn(&mut OutputTable),
    ) {
        let mut registry = test_outputs();
        blank(&mut registry);
        let err = registry
            .check_complete()
            .expect_err("blank topic must fail the completeness check");
        let msg = format!("{err:#}");
        assert!(
            msg.contains(output_name),
            "error should name the missing output '{output_name}': {msg}"
        );
    }
}