Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

16.4 KB
Raw Blame
//! Output producer: maps Stage 1 [`LeafTransition`]s to per-cohort [`CohortMembershipChange`]s.
//!
//! [`map_transition`] fans out to single-leaf cohorts only; composable (multi-leaf) cohorts are
//! emitted by Stage 2.

pub mod batcher;
pub mod cascade;
pub mod kafka;
pub mod marker;
pub mod merge;
pub mod seed;

use async_trait::async_trait;
use chrono::Utc;
use common_kafka::kafka_producer::KafkaProduceError;
use metrics::counter;
use serde::{Deserialize, Serialize};

use cohort_core::seed::RunId;
// Hoisted to the shared seed contract so the seeder's marker watcher and this producer agree on the
// wire bytes; re-exported here so processor call sites keep their `crate::producer` import path.
pub use cohort_core::seed::{ReconcileCompleteMarker, ReconcileMarker, ReconcileWithheldMarker};

use crate::filters::reverse_index::TeamFilters;
use crate::filters::CohortId;
use crate::observability::metrics::OUTPUT_TRANSITIONS_UNMAPPED;
use crate::producer::merge::Capture;
use crate::stage1::transition::{LeafTransition, TransitionKind};

pub use batcher::OutputBuffer;
pub use cascade::{CaptureCascadeSink, CascadeSink, KafkaCascadeSink, NoopCascadeSink};
pub use kafka::KafkaMembershipSink;
pub use marker::{
    CaptureReconcileMarkerSink, KafkaReconcileMarkerSink, NoopReconcileMarkerSink,
    ReconcileMarkerSink,
};
pub use merge::{
    CaptureStreamEventSink, CaptureTransferSink, KafkaStreamEventSink, KafkaTransferSink,
    StreamEventSink, TransferSink,
};
pub use seed::{CaptureSeedTileSink, KafkaSeedTileSink, NoopSeedTileSink, SeedTileSink};

/// One per-cohort membership change on the membership output topic.
///
/// `origin`/`run_id` are additive and absent on the live path, so live emissions stay
/// byte-identical to the pre-field contract.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CohortMembershipChange {
    pub team_id: i32,
    pub cohort_id: i32,
    pub person_id: String,
    /// ClickHouse `DateTime64(6)` wire format.
    pub last_updated: String,
    pub status: MembershipStatus,
    /// Backfill provenance; `None` on the live path.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub origin: Option<ChangeOrigin>,
    /// The backfill run that emitted this change. A cascade keeps `origin` but not `run_id`, because
    /// the membership writer sweeps each (run_id, cohort) and the referrer cohort is not in that run.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub run_id: Option<RunId>,
}

/// Non-live provenance of a membership change; new origins (e.g. a reconcile snapshot) land
/// additively with their producers.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ChangeOrigin {
    Seed,
    Reconcile,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MembershipStatus {
    Entered,
    Left,
}

impl MembershipStatus {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::Entered => "entered",
            Self::Left => "left",
        }
    }
}

impl From<TransitionKind> for MembershipStatus {
    fn from(kind: TransitionKind) -> Self {
        match kind {
            TransitionKind::Entered => Self::Entered,
            TransitionKind::Left => Self::Left,
        }
    }
}

/// Fan out one leaf transition to one membership change per single-leaf cohort that owns its LSK.
pub fn map_transition<'a>(
    filters: &'a TeamFilters,
    transition: &'a LeafTransition,
    last_updated: &'a str,
) -> impl Iterator<Item = CohortMembershipChange> + 'a {
    let cohorts: &[CohortId] = filters
        .by_lsk_to_single_leaf_cohorts
        .get(&transition.leaf_state_key)
        .map_or(&[], Vec::as_slice);
    if cohorts.is_empty()
        && !filters
            .by_lsk_to_composable_cohorts
            .contains_key(&transition.leaf_state_key)
    {
        counter!(OUTPUT_TRANSITIONS_UNMAPPED, "reason" => "no_emitting_cohort").increment(1);
    }
    let team_id = transition.team_id.0;
    let status = MembershipStatus::from(transition.kind);
    let person_id = transition.person_id.to_string();
    cohorts
        .iter()
        .map(move |&cohort_id| CohortMembershipChange {
            team_id,
            cohort_id: cohort_id.0,
            person_id: person_id.clone(),
            last_updated: last_updated.to_string(),
            status,
            origin: None,
            run_id: None,
        })
}

/// Current UTC time as a ClickHouse `DateTime64(6)` string (microseconds).
pub fn now_last_updated() -> String {
    format_last_updated(Utc::now().timestamp_micros())
}

/// Per-partition output-version allocator. Wall time supplies the normal value; the local floor
/// makes every later worker message strictly newer even if the clock stalls or moves backward.
#[derive(Debug, Default)]
pub(crate) struct LastUpdatedClock {
    last_micros: Option<i64>,
}

impl LastUpdatedClock {
    pub fn next(&mut self) -> String {
        self.next_at(Utc::now().timestamp_micros())
    }

    fn next_at(&mut self, observed_micros: i64) -> String {
        let next_micros = self.last_micros.map_or(observed_micros, |last_micros| {
            observed_micros.max(last_micros.saturating_add(1))
        });
        self.last_micros = Some(next_micros);
        format_last_updated(next_micros)
    }
}

fn format_last_updated(timestamp_micros: i64) -> String {
    chrono::DateTime::<Utc>::from_timestamp_micros(timestamp_micros)
        .expect("current UTC timestamps fit Chrono's supported range")
        .format("%Y-%m-%d %H:%M:%S%.6f")
        .to_string()
}

/// `produce` must hand every change to the producer before its future first returns `Pending`, and
/// await only the acks after that. The worker's produce window polls a live batch's produce once
/// and then moves on, so a sink that awaits before it enqueues lets later output overtake the batch.
#[async_trait]
pub trait MembershipSink: Send + Sync {
    async fn produce(
        &self,
        changes: Vec<CohortMembershipChange>,
    ) -> Vec<Result<(), KafkaProduceError>>;
}

#[derive(Debug, Default, Clone)]
pub struct CaptureSink(Capture<CohortMembershipChange>);

impl CaptureSink {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn failing_first(n: usize) -> Self {
        Self(Capture::failing_first(n))
    }

    pub fn changes(&self) -> Vec<CohortMembershipChange> {
        self.0.recorded()
    }

    /// Produce calls made, failed ones included — the round trips a batched apply is amortizing.
    pub fn produce_calls(&self) -> usize {
        self.0.produce_calls()
    }
}

#[async_trait]
impl MembershipSink for CaptureSink {
    async fn produce(
        &self,
        changes: Vec<CohortMembershipChange>,
    ) -> Vec<Result<(), KafkaProduceError>> {
        self.0.produce(changes)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use chrono_tz::UTC;
    use serde_json::{json, Value};
    use uuid::Uuid;

    use crate::filters::{CohortId, TeamFiltersBuilder, TeamId};
    use crate::stage1::key::LeafStateKey;

    const HASH: [u8; 16] = *b"0123456789abcdef";
    const TS: &str = "2026-05-26 12:34:56.789123";

    fn behavioral_leaf(time_value: i64) -> Value {
        json!({
            "type": "behavioral",
            "value": "performed_event",
            "key": "$pageview",
            "time_value": time_value,
            "time_interval": "day",
            "conditionHash": "0123456789abcdef",
            "bytecode": ["_H", 1, 32, "$pageview", 32, "event", 1, 1, 11],
        })
    }

    fn person_leaf() -> Value {
        json!({
            "type": "person",
            "key": "email",
            "value": "[email protected]",
            "operator": "exact",
            "conditionHash": "fedcba9876543210",
            "bytecode": ["_H", 1, 32, "[email protected]", 32, "email", 32, "properties", 32, "person", 1, 3, 11],
        })
    }

    fn wrap(values: Vec<Value>) -> Value {
        json!({ "properties": { "type": "AND", "values": values } })
    }

    fn transition(lsk: LeafStateKey, kind: TransitionKind) -> LeafTransition {
        LeafTransition {
            team_id: TeamId(42),
            leaf_state_key: lsk,
            person_id: Uuid::from_u128(0x01928aaa),
            condition_hash: HASH,
            kind,
        }
    }

    #[test]
    fn map_transition_entered_maps_to_one_change() {
        let mut builder = TeamFiltersBuilder::default();
        builder
            .add_cohort(CohortId(91204), TeamId(42), &wrap(vec![behavioral_leaf(7)]))
            .unwrap();
        let filters = builder.freeze(UTC);
        let lsk = filters.by_condition_to_lsk[&HASH][0];

        let changes: Vec<_> =
            map_transition(&filters, &transition(lsk, TransitionKind::Entered), TS).collect();

        assert_eq!(changes.len(), 1);
        assert_eq!(changes[0].team_id, 42);
        assert_eq!(changes[0].cohort_id, 91204);
        assert_eq!(changes[0].status, MembershipStatus::Entered);
        assert_eq!(changes[0].last_updated, TS);
        assert_eq!(
            changes[0].person_id,
            Uuid::from_u128(0x01928aaa).to_string()
        );
    }

    #[test]
    fn map_transition_left_carries_left_status() {
        let mut builder = TeamFiltersBuilder::default();
        builder
            .add_cohort(CohortId(1), TeamId(42), &wrap(vec![person_leaf()]))
            .unwrap();
        let filters = builder.freeze(UTC);
        let lsk = LeafStateKey::for_person_property(b"fedcba9876543210");

        let changes: Vec<_> =
            map_transition(&filters, &transition(lsk, TransitionKind::Left), TS).collect();

        assert_eq!(changes.len(), 1);
        assert_eq!(changes[0].status, MembershipStatus::Left);
    }

    #[test]
    fn map_transition_fans_out_to_every_owning_cohort() {
        let mut builder = TeamFiltersBuilder::default();
        builder
            .add_cohort(CohortId(1), TeamId(42), &wrap(vec![behavioral_leaf(7)]))
            .unwrap();
        builder
            .add_cohort(CohortId(2), TeamId(42), &wrap(vec![behavioral_leaf(7)]))
            .unwrap();
        let filters = builder.freeze(UTC);
        let lsk = filters.by_condition_to_lsk[&HASH][0];

        let mut cohorts: Vec<i32> =
            map_transition(&filters, &transition(lsk, TransitionKind::Entered), TS)
                .map(|change| change.cohort_id)
                .collect();
        cohorts.sort_unstable();
        assert_eq!(cohorts, vec![1, 2]);
    }

    #[test]
    fn map_transition_for_a_multi_leaf_only_leaf_is_empty() {
        let mut builder = TeamFiltersBuilder::default();
        builder
            .add_cohort(
                CohortId(1),
                TeamId(42),
                &wrap(vec![behavioral_leaf(7), person_leaf()]),
            )
            .unwrap();
        let filters = builder.freeze(UTC);
        let lsk = filters.by_condition_to_lsk[&HASH][0];

        let changes: Vec<_> =
            map_transition(&filters, &transition(lsk, TransitionKind::Entered), TS).collect();
        assert!(
            changes.is_empty(),
            "a leaf owned only by a multi-leaf cohort produces no membership output",
        );
    }

    #[test]
    fn serialized_change_has_exactly_the_contract_keys() {
        let change = CohortMembershipChange {
            team_id: 42,
            cohort_id: 91204,
            person_id: "01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee".to_string(),
            last_updated: TS.to_string(),
            status: MembershipStatus::Entered,
            origin: None,
            run_id: None,
        };
        let value = serde_json::to_value(&change).unwrap();
        let object = value.as_object().unwrap();

        let mut keys: Vec<&str> = object.keys().map(String::as_str).collect();
        keys.sort_unstable();
        assert_eq!(
            keys,
            vec![
                "cohort_id",
                "last_updated",
                "person_id",
                "status",
                "team_id"
            ],
        );
        assert_eq!(object["status"], json!("entered"));
        assert!(object["person_id"].is_string());
        assert_eq!(object["team_id"], json!(42));
    }

    #[test]
    fn left_status_serializes_snake_case() {
        let value = serde_json::to_value(MembershipStatus::Left).unwrap();
        assert_eq!(value, json!("left"));
    }

    #[test]
    fn reconcile_origin_serializes_snake_case() {
        let value = serde_json::to_value(ChangeOrigin::Reconcile).unwrap();
        assert_eq!(value, json!("reconcile"));
    }

    #[test]
    fn live_path_change_stays_byte_identical_and_a_seed_tag_adds_only_the_two_keys() {
        let live = CohortMembershipChange {
            team_id: 42,
            cohort_id: 91204,
            person_id: "01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee".to_string(),
            last_updated: TS.to_string(),
            status: MembershipStatus::Entered,
            origin: None,
            run_id: None,
        };
        // The exact pre-field wire string: downstream JSONEachRow must see no new keys on the
        // live path.
        assert_eq!(
            serde_json::to_string(&live).unwrap(),
            r#"{"team_id":42,"cohort_id":91204,"person_id":"01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee","last_updated":"2026-05-26 12:34:56.789123","status":"entered"}"#,
        );
        // And the pre-field payload still deserializes (defaulting both to None).
        let decoded: CohortMembershipChange =
            serde_json::from_str(&serde_json::to_string(&live).unwrap()).unwrap();
        assert_eq!(decoded, live);

        let tagged = CohortMembershipChange {
            origin: Some(ChangeOrigin::Seed),
            run_id: Some(RunId(uuid::Uuid::nil())),
            ..live
        };
        let value = serde_json::to_value(&tagged).unwrap();
        assert_eq!(value["origin"], json!("seed"));
        assert_eq!(
            value["run_id"],
            json!("00000000-0000-0000-0000-000000000000")
        );
        assert_eq!(value.as_object().unwrap().len(), 7);
    }

    #[test]
    fn now_last_updated_is_microsecond_clickhouse_format() {
        let now = now_last_updated();
        let (date_time, fraction) = now.split_once('.').expect("has a fractional part");
        assert_eq!(fraction.len(), 6, "microseconds, not nanoseconds: {now}");
        assert!(fraction.chars().all(|c| c.is_ascii_digit()));

        let (date, time) = date_time.split_once(' ').expect("date and time");
        assert_eq!(date.len(), 10);
        assert_eq!(date.matches('-').count(), 2);
        assert_eq!(time.matches(':').count(), 2);
    }

    #[test]
    fn last_updated_clock_is_strictly_monotonic_when_wall_time_stalls_or_rewinds() {
        let mut clock = LastUpdatedClock::default();

        let first = clock.next_at(1_700_000_000_000_000);
        let stalled = clock.next_at(1_700_000_000_000_000);
        let rewound = clock.next_at(1_699_999_999_000_000);

        assert!(first < stalled);
        assert!(stalled < rewound);
        assert_eq!(first, "2023-11-14 22:13:20.000000");
        assert_eq!(stalled, "2023-11-14 22:13:20.000001");
        assert_eq!(rewound, "2023-11-14 22:13:20.000002");
    }

    #[tokio::test]
    async fn capture_sink_records_and_acks_in_order() {
        let sink = CaptureSink::new();
        let change = CohortMembershipChange {
            team_id: 1,
            cohort_id: 2,
            person_id: "p".to_string(),
            last_updated: TS.to_string(),
            status: MembershipStatus::Entered,
            origin: None,
            run_id: None,
        };
        let acks = sink.produce(vec![change.clone(), change.clone()]).await;
        assert_eq!(acks.len(), 2);
        assert!(acks.iter().all(Result::is_ok));
        assert_eq!(sink.changes(), vec![change.clone(), change]);
    }

    #[tokio::test]
    async fn capture_sink_failing_first_fails_then_succeeds_recording_nothing_on_failure() {
        let sink = CaptureSink::failing_first(1);
        let change = CohortMembershipChange {
            team_id: 1,
            cohort_id: 2,
            person_id: "p".to_string(),
            last_updated: TS.to_string(),
            status: MembershipStatus::Left,
            origin: None,
            run_id: None,
        };

        let first = sink.produce(vec![change.clone()]).await;
        assert!(first.iter().all(Result::is_err), "first flush fails");
        assert!(sink.changes().is_empty(), "a failed flush records nothing");

        let second = sink.produce(vec![change.clone()]).await;
        assert!(second.iter().all(Result::is_ok), "second flush succeeds");
        assert_eq!(sink.changes(), vec![change]);
    }
}