:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Binds every fixed routing [`Destination`] to its output: a Kafka topic and
//! the named producer that carries it.
use std::sync::Arc;
use crate::config::OutputsConfig;
use crate::pipeline::{Address, AiLane, AnalyticsLane, BasicLane, PipelineLane, SessionReplayLane};
use crate::producers::ProducerName;
/// Which configured output a routing decision selects, named by pipeline and
/// lane. Distinct outputs may share a topic.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Destination {
AnalyticsMain,
AnalyticsOverflow,
AnalyticsHistorical,
ClientWarningsMain,
HeatmapsMain,
SessionReplayMain,
SessionReplayOverflow,
/// Every pipeline's dlq lane, one shared output.
Dlq,
ErrorTrackingMain,
AiMain,
AiOverflow,
/// Admin-configured topic copied from `redirect_to_topic`.
Custom(String),
}
impl Destination {
/// Every registered always-required output. `check_complete` walks this so
/// a newly added output is caught at boot rather than at first produce.
const REGISTERED: [Destination; 11] = [
Destination::AnalyticsMain,
Destination::AnalyticsOverflow,
Destination::AnalyticsHistorical,
Destination::ClientWarningsMain,
Destination::HeatmapsMain,
Destination::SessionReplayMain,
Destination::SessionReplayOverflow,
Destination::Dlq,
Destination::ErrorTrackingMain,
Destination::AiMain,
Destination::AiOverflow,
];
pub(crate) fn for_lane(lane: PipelineLane) -> Destination {
match lane {
PipelineLane::Analytics(AnalyticsLane::Main) => Destination::AnalyticsMain,
PipelineLane::Analytics(AnalyticsLane::Overflow) => Destination::AnalyticsOverflow,
PipelineLane::Analytics(AnalyticsLane::Historical) => Destination::AnalyticsHistorical,
PipelineLane::Ai(AiLane::Main) => Destination::AiMain,
PipelineLane::Ai(AiLane::Overflow) => Destination::AiOverflow,
PipelineLane::Warnings(BasicLane::Main) => Destination::ClientWarningsMain,
PipelineLane::Heatmaps(BasicLane::Main) => Destination::HeatmapsMain,
PipelineLane::ErrorTracking(BasicLane::Main) => Destination::ErrorTrackingMain,
PipelineLane::Replay(SessionReplayLane::Main) => Destination::SessionReplayMain,
PipelineLane::Replay(SessionReplayLane::Overflow) => Destination::SessionReplayOverflow,
}
}
pub(crate) fn for_address(address: Address) -> Destination {
match address {
Address::Lane(lane) => Self::for_lane(lane),
Address::Dlq => Destination::Dlq,
Address::Custom(topic) => Destination::Custom(topic),
}
}
/// Whether this output participates in the boot completeness check.
/// Deliberately exhaustive: a new variant cannot compile without
/// declaring which side it is on, and
/// `registered_is_exactly_the_required_outputs` pins [`Self::REGISTERED`]
/// to the `true` arms so declaring `true` without joining the array
/// fails a test instead of silently skipping the check.
#[cfg(test)]
fn is_required(&self) -> bool {
match self {
Destination::AnalyticsMain
| Destination::AnalyticsOverflow
| Destination::AnalyticsHistorical
| Destination::ClientWarningsMain
| Destination::HeatmapsMain
| Destination::SessionReplayMain
| Destination::SessionReplayOverflow
| Destination::Dlq
| Destination::ErrorTrackingMain
| Destination::AiMain
| Destination::AiOverflow => true,
// A custom redirect carries its own topic per event.
Destination::Custom(_) => false,
}
}
/// Stable, low-cardinality label for diagnostics. `Custom` collapses to
/// "custom" so admin topic names never leak into error messages.
fn name(&self) -> &'static str {
match self {
Destination::AnalyticsMain => "analytics-main",
Destination::AnalyticsOverflow => "analytics-overflow",
Destination::AnalyticsHistorical => "analytics-historical",
Destination::ClientWarningsMain => "clientwarnings-main",
Destination::HeatmapsMain => "heatmaps-main",
Destination::SessionReplayMain => "sessionreplay-main",
Destination::SessionReplayOverflow => "sessionreplay-overflow",
Destination::Dlq => "dlq",
Destination::ErrorTrackingMain => "errortracking-main",
Destination::AiMain => "ai-main",
Destination::AiOverflow => "ai-overflow",
Destination::Custom(_) => "custom",
}
}
}
/// `R` is how the target names its producer: a [`ProducerName`] in config,
/// the producer handle once setup resolves it.
#[derive(Clone, Debug)]
pub struct OutputTarget<R = ProducerName> {
// `Arc<str>` so the per-record lookup, metric labels and produce record never allocate.
pub(crate) topic: Arc<str>,
pub(crate) producer: R,
}
#[derive(Clone, Debug)]
pub struct OutputTable<R = ProducerName> {
pub(crate) analytics_main: OutputTarget<R>,
pub(crate) analytics_overflow: OutputTarget<R>,
pub(crate) analytics_historical: OutputTarget<R>,
pub(crate) session_replay_main: OutputTarget<R>,
pub(crate) session_replay_overflow: OutputTarget<R>,
pub(crate) client_warnings: OutputTarget<R>,
pub(crate) heatmaps: OutputTarget<R>,
pub(crate) dlq: OutputTarget<R>,
pub(crate) error_tracking: OutputTarget<R>,
pub(crate) ai_main: OutputTarget<R>,
pub(crate) ai_overflow: OutputTarget<R>,
pub(crate) has_ai_overflow_lane: bool,
pub(crate) custom_producer: R,
}
impl<R> OutputTable<R> {
fn target_for(&self, output: &Destination) -> Option<&OutputTarget<R>> {
match output {
Destination::AnalyticsMain => Some(&self.analytics_main),
Destination::AnalyticsOverflow => Some(&self.analytics_overflow),
Destination::AnalyticsHistorical => Some(&self.analytics_historical),
Destination::ClientWarningsMain => Some(&self.client_warnings),
Destination::HeatmapsMain => Some(&self.heatmaps),
Destination::SessionReplayMain => Some(&self.session_replay_main),
Destination::SessionReplayOverflow => Some(&self.session_replay_overflow),
Destination::Dlq => Some(&self.dlq),
Destination::ErrorTrackingMain => Some(&self.error_tracking),
Destination::AiMain => Some(&self.ai_main),
Destination::AiOverflow => Some(&self.ai_overflow),
Destination::Custom(_) => None,
}
}
pub fn topic_for<'a>(&'a self, output: &'a Destination) -> &'a str {
match (output, self.target_for(output)) {
(Destination::Custom(topic), _) => topic,
(_, Some(target)) => &target.topic,
(_, None) => unreachable!("every fixed output has a target"),
}
}
pub(crate) fn resolve(&self, output: &Destination) -> (Arc<str>, &R) {
match (output, self.target_for(output)) {
(_, Some(target)) => (Arc::clone(&target.topic), &target.producer),
(Destination::Custom(topic), None) => {
(Arc::from(topic.as_str()), &self.custom_producer)
}
(_, None) => unreachable!("every fixed output has a target"),
}
}
pub fn map_producers<T>(&self, mut f: impl FnMut(&R) -> T) -> OutputTable<T> {
let custom_producer = f(&self.custom_producer);
let mut target = |t: &OutputTarget<R>| OutputTarget {
topic: Arc::clone(&t.topic),
producer: f(&t.producer),
};
OutputTable {
analytics_main: target(&self.analytics_main),
analytics_overflow: target(&self.analytics_overflow),
analytics_historical: target(&self.analytics_historical),
session_replay_main: target(&self.session_replay_main),
session_replay_overflow: target(&self.session_replay_overflow),
client_warnings: target(&self.client_warnings),
heatmaps: target(&self.heatmaps),
dlq: target(&self.dlq),
error_tracking: target(&self.error_tracking),
ai_main: target(&self.ai_main),
ai_overflow: target(&self.ai_overflow),
has_ai_overflow_lane: self.has_ai_overflow_lane,
custom_producer,
}
}
pub fn has_ai_overflow_lane(&self) -> bool {
self.has_ai_overflow_lane
}
/// Every registered output must resolve to a non-empty topic. `Custom` is
/// excluded because it carries its own topic per event.
pub fn check_complete(&self) -> anyhow::Result<()> {
for output in &Destination::REGISTERED {
anyhow::ensure!(
!self.topic_for(output).is_empty(),
"output '{}' resolves to an empty Kafka topic; every non-custom \
output must be bound to a configured, non-empty topic",
output.name(),
);
}
Ok(())
}
}
impl OutputTable {
pub fn new(config: &OutputsConfig, has_ai_overflow_lane: bool) -> Self {
let target = |topic: &String, producer: ProducerName| OutputTarget {
topic: Arc::from(topic.as_str()),
producer,
};
Self {
analytics_main: target(&config.analytics_main_topic, config.analytics_main_producer),
analytics_overflow: target(
&config.analytics_overflow_topic,
config.analytics_overflow_producer,
),
analytics_historical: target(
&config.analytics_historical_topic,
config.analytics_historical_producer,
),
session_replay_main: target(
&config.session_replay_main_topic,
config.session_replay_main_producer,
),
session_replay_overflow: target(
&config.session_replay_overflow_topic,
config.session_replay_overflow_producer,
),
client_warnings: target(
&config.client_warnings_topic,
config.client_warnings_producer,
),
heatmaps: target(&config.heatmaps_topic, config.heatmaps_producer),
dlq: target(&config.dlq_topic, config.dlq_producer),
error_tracking: target(&config.error_tracking_topic, config.error_tracking_producer),
ai_main: target(&config.ai_main_topic, config.ai_main_producer),
ai_overflow: target(&config.ai_overflow_topic, config.ai_overflow_producer),
has_ai_overflow_lane,
custom_producer: config.custom_producer,
}
}
}
#[cfg(test)]
pub(crate) fn test_outputs() -> OutputTable {
let target = |topic: &str| OutputTarget {
topic: Arc::from(topic),
producer: ProducerName::Ingestion,
};
OutputTable {
analytics_main: target("events_plugin_ingestion"),
analytics_overflow: target("events_plugin_ingestion_overflow"),
analytics_historical: target("events_plugin_ingestion_historical"),
session_replay_main: target("events_plugin_ingestion"),
session_replay_overflow: target("replay_overflow"),
client_warnings: target("client_ingestion_warning"),
heatmaps: target("heatmaps"),
dlq: target("events_plugin_ingestion_dlq"),
error_tracking: target("error_tracking_events"),
ai_main: target("ai_events"),
ai_overflow: target("ai_events_overflow"),
has_ai_overflow_lane: true,
custom_producer: ProducerName::Ingestion,
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
#[rstest]
#[case(Destination::AnalyticsMain, "events_plugin_ingestion")]
#[case(Destination::SessionReplayMain, "events_plugin_ingestion")]
#[case(Destination::AnalyticsOverflow, "events_plugin_ingestion_overflow")]
#[case(Destination::AnalyticsHistorical, "events_plugin_ingestion_historical")]
#[case(Destination::ClientWarningsMain, "client_ingestion_warning")]
#[case(Destination::HeatmapsMain, "heatmaps")]
#[case(Destination::SessionReplayOverflow, "replay_overflow")]
#[case(Destination::Dlq, "events_plugin_ingestion_dlq")]
#[case(Destination::ErrorTrackingMain, "error_tracking_events")]
#[case(Destination::AiMain, "ai_events")]
#[case(Destination::AiOverflow, "ai_events_overflow")]
fn topic_for_resolves_registered_outputs(#[case] output: Destination, #[case] expected: &str) {
assert_eq!(test_outputs().topic_for(&output), expected);
}
#[test]
fn topic_for_custom_returns_inline_topic() {
let registry = test_outputs();
assert_eq!(
registry.topic_for(&Destination::Custom("admin_topic".to_string())),
"admin_topic"
);
}
#[test]
fn session_replay_main_does_not_share_analytics_main() {
let mut registry = test_outputs();
registry.session_replay_main.topic = Arc::from("replay_main");
assert_eq!(
registry.topic_for(&Destination::SessionReplayMain),
"replay_main"
);
assert_eq!(
registry.topic_for(&Destination::AnalyticsMain),
"events_plugin_ingestion"
);
}
#[test]
fn each_output_resolves_to_its_own_producer() {
let mut next = 0;
let table = test_outputs().map_producers(|_| {
next += 1;
next
});
let custom = Destination::Custom("admin_topic".to_string());
let (topic, _) = table.resolve(&custom);
assert_eq!(&*topic, "admin_topic");
let producers: std::collections::HashSet<i32> = Destination::REGISTERED
.iter()
.chain([&custom])
.map(|output| *table.resolve(output).1)
.collect();
assert_eq!(producers.len(), Destination::REGISTERED.len() + 1);
}
#[test]
fn check_complete_accepts_full_registry() {
assert!(test_outputs().check_complete().is_ok());
}
/// The all-variants list below is hand-maintained: grow it with the enum.
#[test]
fn registered_is_exactly_the_required_outputs() {
let all = [
Destination::AnalyticsMain,
Destination::AnalyticsOverflow,
Destination::AnalyticsHistorical,
Destination::ClientWarningsMain,
Destination::HeatmapsMain,
Destination::SessionReplayMain,
Destination::SessionReplayOverflow,
Destination::Dlq,
Destination::ErrorTrackingMain,
Destination::AiMain,
Destination::AiOverflow,
Destination::Custom("t".to_string()),
];
for output in &all {
assert_eq!(
Destination::REGISTERED.contains(output),
output.is_required(),
"'{}' must be in REGISTERED exactly when it is required",
output.name()
);
}
}
/// Every registered output, blanked one at a time, must fail the check and
/// the error must name the offending output.
#[rstest]
#[case("analytics-main", |r: &mut OutputTable| r.analytics_main.topic = Arc::from(""))]
#[case("analytics-overflow", |r: &mut OutputTable| r.analytics_overflow.topic = Arc::from(""))]
#[case("analytics-historical", |r: &mut OutputTable| r.analytics_historical.topic = Arc::from(""))]
#[case("clientwarnings-main", |r: &mut OutputTable| r.client_warnings.topic = Arc::from(""))]
#[case("heatmaps-main", |r: &mut OutputTable| r.heatmaps.topic = Arc::from(""))]
#[case("sessionreplay-main", |r: &mut OutputTable| r.session_replay_main.topic = Arc::from(""))]
#[case("sessionreplay-overflow", |r: &mut OutputTable| r.session_replay_overflow.topic = Arc::from(""))]
#[case("dlq", |r: &mut OutputTable| r.dlq.topic = Arc::from(""))]
#[case("errortracking-main", |r: &mut OutputTable| r.error_tracking.topic = Arc::from(""))]
#[case("ai-main", |r: &mut OutputTable| r.ai_main.topic = Arc::from(""))]
#[case("ai-overflow", |r: &mut OutputTable| r.ai_overflow.topic = Arc::from(""))]
fn check_complete_rejects_empty_topic(
#[case] output_name: &str,
#[case] blank: fn(&mut OutputTable),
) {
let mut registry = test_outputs();
blank(&mut registry);
let err = registry
.check_complete()
.expect_err("blank topic must fail the completeness check");
let msg = format!("{err:#}");
assert!(
msg.contains(output_name),
"error should name the missing output '{output_name}': {msg}"
);
}
}