:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Output producer: maps Stage 1 [`LeafTransition`]s to per-cohort [`CohortMembershipChange`]s.
//!
//! [`map_transition`] fans out to single-leaf cohorts only; composable (multi-leaf) cohorts are
//! emitted by Stage 2.
pub mod batcher;
pub mod cascade;
pub mod kafka;
pub mod marker;
pub mod merge;
pub mod seed;
use async_trait::async_trait;
use chrono::Utc;
use common_kafka::kafka_producer::KafkaProduceError;
use metrics::counter;
use serde::{Deserialize, Serialize};
use cohort_core::seed::RunId;
// Hoisted to the shared seed contract so the seeder's marker watcher and this producer agree on the
// wire bytes; re-exported here so processor call sites keep their `crate::producer` import path.
pub use cohort_core::seed::{ReconcileCompleteMarker, ReconcileMarker, ReconcileWithheldMarker};
use crate::filters::reverse_index::TeamFilters;
use crate::filters::CohortId;
use crate::observability::metrics::OUTPUT_TRANSITIONS_UNMAPPED;
use crate::producer::merge::Capture;
use crate::stage1::transition::{LeafTransition, TransitionKind};
pub use batcher::OutputBuffer;
pub use cascade::{CaptureCascadeSink, CascadeSink, KafkaCascadeSink, NoopCascadeSink};
pub use kafka::KafkaMembershipSink;
pub use marker::{
CaptureReconcileMarkerSink, KafkaReconcileMarkerSink, NoopReconcileMarkerSink,
ReconcileMarkerSink,
};
pub use merge::{
CaptureStreamEventSink, CaptureTransferSink, KafkaStreamEventSink, KafkaTransferSink,
StreamEventSink, TransferSink,
};
pub use seed::{CaptureSeedTileSink, KafkaSeedTileSink, NoopSeedTileSink, SeedTileSink};
/// One per-cohort membership change on the membership output topic.
///
/// `origin`/`run_id` are additive and absent on the live path, so live emissions stay
/// byte-identical to the pre-field contract.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CohortMembershipChange {
pub team_id: i32,
pub cohort_id: i32,
pub person_id: String,
/// ClickHouse `DateTime64(6)` wire format.
pub last_updated: String,
pub status: MembershipStatus,
/// Backfill provenance; `None` on the live path.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub origin: Option<ChangeOrigin>,
/// The backfill run that emitted this change. A cascade keeps `origin` but not `run_id`, because
/// the membership writer sweeps each (run_id, cohort) and the referrer cohort is not in that run.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub run_id: Option<RunId>,
}
/// Non-live provenance of a membership change; new origins (e.g. a reconcile snapshot) land
/// additively with their producers.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ChangeOrigin {
Seed,
Reconcile,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MembershipStatus {
Entered,
Left,
}
impl MembershipStatus {
pub fn as_str(self) -> &'static str {
match self {
Self::Entered => "entered",
Self::Left => "left",
}
}
}
impl From<TransitionKind> for MembershipStatus {
fn from(kind: TransitionKind) -> Self {
match kind {
TransitionKind::Entered => Self::Entered,
TransitionKind::Left => Self::Left,
}
}
}
/// Fan out one leaf transition to one membership change per single-leaf cohort that owns its LSK.
pub fn map_transition<'a>(
filters: &'a TeamFilters,
transition: &'a LeafTransition,
last_updated: &'a str,
) -> impl Iterator<Item = CohortMembershipChange> + 'a {
let cohorts: &[CohortId] = filters
.by_lsk_to_single_leaf_cohorts
.get(&transition.leaf_state_key)
.map_or(&[], Vec::as_slice);
if cohorts.is_empty()
&& !filters
.by_lsk_to_composable_cohorts
.contains_key(&transition.leaf_state_key)
{
counter!(OUTPUT_TRANSITIONS_UNMAPPED, "reason" => "no_emitting_cohort").increment(1);
}
let team_id = transition.team_id.0;
let status = MembershipStatus::from(transition.kind);
let person_id = transition.person_id.to_string();
cohorts
.iter()
.map(move |&cohort_id| CohortMembershipChange {
team_id,
cohort_id: cohort_id.0,
person_id: person_id.clone(),
last_updated: last_updated.to_string(),
status,
origin: None,
run_id: None,
})
}
/// Current UTC time as a ClickHouse `DateTime64(6)` string (microseconds).
pub fn now_last_updated() -> String {
format_last_updated(Utc::now().timestamp_micros())
}
/// Per-partition output-version allocator. Wall time supplies the normal value; the local floor
/// makes every later worker message strictly newer even if the clock stalls or moves backward.
#[derive(Debug, Default)]
pub(crate) struct LastUpdatedClock {
last_micros: Option<i64>,
}
impl LastUpdatedClock {
pub fn next(&mut self) -> String {
self.next_at(Utc::now().timestamp_micros())
}
fn next_at(&mut self, observed_micros: i64) -> String {
let next_micros = self.last_micros.map_or(observed_micros, |last_micros| {
observed_micros.max(last_micros.saturating_add(1))
});
self.last_micros = Some(next_micros);
format_last_updated(next_micros)
}
}
fn format_last_updated(timestamp_micros: i64) -> String {
chrono::DateTime::<Utc>::from_timestamp_micros(timestamp_micros)
.expect("current UTC timestamps fit Chrono's supported range")
.format("%Y-%m-%d %H:%M:%S%.6f")
.to_string()
}
/// `produce` must hand every change to the producer before its future first returns `Pending`, and
/// await only the acks after that. The worker's produce window polls a live batch's produce once
/// and then moves on, so a sink that awaits before it enqueues lets later output overtake the batch.
#[async_trait]
pub trait MembershipSink: Send + Sync {
async fn produce(
&self,
changes: Vec<CohortMembershipChange>,
) -> Vec<Result<(), KafkaProduceError>>;
}
#[derive(Debug, Default, Clone)]
pub struct CaptureSink(Capture<CohortMembershipChange>);
impl CaptureSink {
pub fn new() -> Self {
Self::default()
}
pub fn failing_first(n: usize) -> Self {
Self(Capture::failing_first(n))
}
pub fn changes(&self) -> Vec<CohortMembershipChange> {
self.0.recorded()
}
/// Produce calls made, failed ones included — the round trips a batched apply is amortizing.
pub fn produce_calls(&self) -> usize {
self.0.produce_calls()
}
}
#[async_trait]
impl MembershipSink for CaptureSink {
async fn produce(
&self,
changes: Vec<CohortMembershipChange>,
) -> Vec<Result<(), KafkaProduceError>> {
self.0.produce(changes)
}
}
#[cfg(test)]
mod tests {
use super::*;
use chrono_tz::UTC;
use serde_json::{json, Value};
use uuid::Uuid;
use crate::filters::{CohortId, TeamFiltersBuilder, TeamId};
use crate::stage1::key::LeafStateKey;
const HASH: [u8; 16] = *b"0123456789abcdef";
const TS: &str = "2026-05-26 12:34:56.789123";
fn behavioral_leaf(time_value: i64) -> Value {
json!({
"type": "behavioral",
"value": "performed_event",
"key": "$pageview",
"time_value": time_value,
"time_interval": "day",
"conditionHash": "0123456789abcdef",
"bytecode": ["_H", 1, 32, "$pageview", 32, "event", 1, 1, 11],
})
}
fn person_leaf() -> Value {
json!({
"type": "person",
"key": "email",
"value": "[email protected]",
"operator": "exact",
"conditionHash": "fedcba9876543210",
"bytecode": ["_H", 1, 32, "[email protected]", 32, "email", 32, "properties", 32, "person", 1, 3, 11],
})
}
fn wrap(values: Vec<Value>) -> Value {
json!({ "properties": { "type": "AND", "values": values } })
}
fn transition(lsk: LeafStateKey, kind: TransitionKind) -> LeafTransition {
LeafTransition {
team_id: TeamId(42),
leaf_state_key: lsk,
person_id: Uuid::from_u128(0x01928aaa),
condition_hash: HASH,
kind,
}
}
#[test]
fn map_transition_entered_maps_to_one_change() {
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(CohortId(91204), TeamId(42), &wrap(vec![behavioral_leaf(7)]))
.unwrap();
let filters = builder.freeze(UTC);
let lsk = filters.by_condition_to_lsk[&HASH][0];
let changes: Vec<_> =
map_transition(&filters, &transition(lsk, TransitionKind::Entered), TS).collect();
assert_eq!(changes.len(), 1);
assert_eq!(changes[0].team_id, 42);
assert_eq!(changes[0].cohort_id, 91204);
assert_eq!(changes[0].status, MembershipStatus::Entered);
assert_eq!(changes[0].last_updated, TS);
assert_eq!(
changes[0].person_id,
Uuid::from_u128(0x01928aaa).to_string()
);
}
#[test]
fn map_transition_left_carries_left_status() {
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(CohortId(1), TeamId(42), &wrap(vec![person_leaf()]))
.unwrap();
let filters = builder.freeze(UTC);
let lsk = LeafStateKey::for_person_property(b"fedcba9876543210");
let changes: Vec<_> =
map_transition(&filters, &transition(lsk, TransitionKind::Left), TS).collect();
assert_eq!(changes.len(), 1);
assert_eq!(changes[0].status, MembershipStatus::Left);
}
#[test]
fn map_transition_fans_out_to_every_owning_cohort() {
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(CohortId(1), TeamId(42), &wrap(vec![behavioral_leaf(7)]))
.unwrap();
builder
.add_cohort(CohortId(2), TeamId(42), &wrap(vec![behavioral_leaf(7)]))
.unwrap();
let filters = builder.freeze(UTC);
let lsk = filters.by_condition_to_lsk[&HASH][0];
let mut cohorts: Vec<i32> =
map_transition(&filters, &transition(lsk, TransitionKind::Entered), TS)
.map(|change| change.cohort_id)
.collect();
cohorts.sort_unstable();
assert_eq!(cohorts, vec![1, 2]);
}
#[test]
fn map_transition_for_a_multi_leaf_only_leaf_is_empty() {
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(
CohortId(1),
TeamId(42),
&wrap(vec![behavioral_leaf(7), person_leaf()]),
)
.unwrap();
let filters = builder.freeze(UTC);
let lsk = filters.by_condition_to_lsk[&HASH][0];
let changes: Vec<_> =
map_transition(&filters, &transition(lsk, TransitionKind::Entered), TS).collect();
assert!(
changes.is_empty(),
"a leaf owned only by a multi-leaf cohort produces no membership output",
);
}
#[test]
fn serialized_change_has_exactly_the_contract_keys() {
let change = CohortMembershipChange {
team_id: 42,
cohort_id: 91204,
person_id: "01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee".to_string(),
last_updated: TS.to_string(),
status: MembershipStatus::Entered,
origin: None,
run_id: None,
};
let value = serde_json::to_value(&change).unwrap();
let object = value.as_object().unwrap();
let mut keys: Vec<&str> = object.keys().map(String::as_str).collect();
keys.sort_unstable();
assert_eq!(
keys,
vec![
"cohort_id",
"last_updated",
"person_id",
"status",
"team_id"
],
);
assert_eq!(object["status"], json!("entered"));
assert!(object["person_id"].is_string());
assert_eq!(object["team_id"], json!(42));
}
#[test]
fn left_status_serializes_snake_case() {
let value = serde_json::to_value(MembershipStatus::Left).unwrap();
assert_eq!(value, json!("left"));
}
#[test]
fn reconcile_origin_serializes_snake_case() {
let value = serde_json::to_value(ChangeOrigin::Reconcile).unwrap();
assert_eq!(value, json!("reconcile"));
}
#[test]
fn live_path_change_stays_byte_identical_and_a_seed_tag_adds_only_the_two_keys() {
let live = CohortMembershipChange {
team_id: 42,
cohort_id: 91204,
person_id: "01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee".to_string(),
last_updated: TS.to_string(),
status: MembershipStatus::Entered,
origin: None,
run_id: None,
};
// The exact pre-field wire string: downstream JSONEachRow must see no new keys on the
// live path.
assert_eq!(
serde_json::to_string(&live).unwrap(),
r#"{"team_id":42,"cohort_id":91204,"person_id":"01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee","last_updated":"2026-05-26 12:34:56.789123","status":"entered"}"#,
);
// And the pre-field payload still deserializes (defaulting both to None).
let decoded: CohortMembershipChange =
serde_json::from_str(&serde_json::to_string(&live).unwrap()).unwrap();
assert_eq!(decoded, live);
let tagged = CohortMembershipChange {
origin: Some(ChangeOrigin::Seed),
run_id: Some(RunId(uuid::Uuid::nil())),
..live
};
let value = serde_json::to_value(&tagged).unwrap();
assert_eq!(value["origin"], json!("seed"));
assert_eq!(
value["run_id"],
json!("00000000-0000-0000-0000-000000000000")
);
assert_eq!(value.as_object().unwrap().len(), 7);
}
#[test]
fn now_last_updated_is_microsecond_clickhouse_format() {
let now = now_last_updated();
let (date_time, fraction) = now.split_once('.').expect("has a fractional part");
assert_eq!(fraction.len(), 6, "microseconds, not nanoseconds: {now}");
assert!(fraction.chars().all(|c| c.is_ascii_digit()));
let (date, time) = date_time.split_once(' ').expect("date and time");
assert_eq!(date.len(), 10);
assert_eq!(date.matches('-').count(), 2);
assert_eq!(time.matches(':').count(), 2);
}
#[test]
fn last_updated_clock_is_strictly_monotonic_when_wall_time_stalls_or_rewinds() {
let mut clock = LastUpdatedClock::default();
let first = clock.next_at(1_700_000_000_000_000);
let stalled = clock.next_at(1_700_000_000_000_000);
let rewound = clock.next_at(1_699_999_999_000_000);
assert!(first < stalled);
assert!(stalled < rewound);
assert_eq!(first, "2023-11-14 22:13:20.000000");
assert_eq!(stalled, "2023-11-14 22:13:20.000001");
assert_eq!(rewound, "2023-11-14 22:13:20.000002");
}
#[tokio::test]
async fn capture_sink_records_and_acks_in_order() {
let sink = CaptureSink::new();
let change = CohortMembershipChange {
team_id: 1,
cohort_id: 2,
person_id: "p".to_string(),
last_updated: TS.to_string(),
status: MembershipStatus::Entered,
origin: None,
run_id: None,
};
let acks = sink.produce(vec![change.clone(), change.clone()]).await;
assert_eq!(acks.len(), 2);
assert!(acks.iter().all(Result::is_ok));
assert_eq!(sink.changes(), vec![change.clone(), change]);
}
#[tokio::test]
async fn capture_sink_failing_first_fails_then_succeeds_recording_nothing_on_failure() {
let sink = CaptureSink::failing_first(1);
let change = CohortMembershipChange {
team_id: 1,
cohort_id: 2,
person_id: "p".to_string(),
last_updated: TS.to_string(),
status: MembershipStatus::Left,
origin: None,
run_id: None,
};
let first = sink.produce(vec![change.clone()]).await;
assert!(first.iter().all(Result::is_err), "first flush fails");
assert!(sink.changes().is_empty(), "a failed flush records nothing");
let second = sink.produce(vec![change.clone()]).await;
assert!(second.iter().all(Result::is_ok), "second flush succeeds");
assert_eq!(sink.changes(), vec![change]);
}
}