:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result};
use common_database::get_pool_with_config;
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Handle, Manager};
use rdkafka::consumer::{Consumer, ConsumerContext, StreamConsumer};
use tokio::net::TcpListener;
use tokio::signal::unix::{signal, SignalKind};
use tracing::{info, warn};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{fmt, EnvFilter, Layer};
use cohort_stream_processor::config::Config;
use cohort_stream_processor::consumers::{
BootReadiness, CascadeRoute, CohortStreamEventsConsumer, EventDispatcher, FollowerConsumer,
FollowerRoute, MergeRoute, SeedFollowerConsumer, TransferRoute,
};
use cohort_stream_processor::filters::CatalogHandle;
use cohort_stream_processor::merge::gc::MergeGcSweeper;
use cohort_stream_processor::merge::redrive::RedriveSweeper;
use cohort_stream_processor::observability;
use cohort_stream_processor::observability::disk::SharedDiskUtilization;
use cohort_stream_processor::observability::store_files::StoreFilesSweeper;
use cohort_stream_processor::observability::store_stats::{DiskProbe, StoreStatsSweeper};
use cohort_stream_processor::observability::tokio_monitor::TokioRuntimeMonitor;
use cohort_stream_processor::partitions::{
AdmittedSplit, CohortConsumerContext, ConsumerPauser, Follower, FollowerSet, InputGroups,
LiveWatermarks, OffsetTracker, OwnershipAudit, PartitionPauser, PartitionRouter,
AUDIT_INTERVAL,
};
use cohort_stream_processor::producer::{
CascadeSink, KafkaCascadeSink, KafkaMembershipSink, KafkaReconcileMarkerSink,
KafkaSeedTileSink, KafkaStreamEventSink, KafkaTransferSink, MembershipSink, NoopCascadeSink,
NoopReconcileMarkerSink, NoopSeedTileSink, ReconcileMarkerSink, SeedTileSink, StreamEventSink,
TransferSink,
};
use cohort_stream_processor::store::durability::checkpoint::run_checkpoints;
use cohort_stream_processor::store::durability::{
ensure_one_filesystem, open_store, upload_cadence, CheckpointSweeper,
};
use cohort_stream_processor::store::StoreHandle;
use cohort_stream_processor::sweep::{
run_sweep_loop, run_sweep_loop_delayed, DispatchSweeper, ReconcileDrainSweeper,
};
use cohort_stream_processor::workers::{
MergeWorkerDeps, PersonSeedDeps, ReconcileBacklog, ReconcileDeps,
};
// Under `hotpath-alloc`, `#[hotpath::main]` installs its counting allocator around jemalloc instead.
#[cfg(not(feature = "hotpath-alloc"))]
common_alloc::used!();
const SERVICE_NAME: &str = "cohort-stream-processor";
#[hotpath::main(allocator = common_alloc::DefaultAllocator)]
fn main() -> Result<()> {
let config = Config::init_from_env()
.context("Failed to load configuration from environment variables")?;
let mut runtime_builder = tokio::runtime::Builder::new_multi_thread();
runtime_builder.enable_all();
if config.tokio_worker_threads > 0 {
runtime_builder.worker_threads(config.tokio_worker_threads);
}
let runtime = runtime_builder
.build()
.context("Failed to build tokio runtime")?;
hotpath::tokio_runtime!(runtime.handle());
runtime.block_on(async_main(config))
}
async fn async_main(config: Config) -> Result<()> {
init_tracing();
log_startup(&config);
config.validate_startup()?;
let claim = config.split_claim()?;
let lineage = config
.checkpoint_enabled
.then(|| config.checkpoint_lineage())
.transpose()?;
// `kill -USR1 1` uploads a checkpoint ahead of a planned stop, such as a reshard, because the
// final checkpoint stays local. The handler goes in before the restore, so a request made
// during boot waits for the checkpoint loop: without a handler, PID 1 discards the signal.
let upload_requests = lineage
.map(|_| signal(SignalKind::user_defined1()))
.transpose()
.context("installing the SIGUSR1 handler")?;
// Covers the 45 s follower windows, then the checkpoint component's 60 s. The pod's termination
// grace period must exceed it plus the chart's 5 s preStop sleep, so above 115 s, or Kubernetes
// cuts the final checkpoint off.
let mut manager = Manager::builder(SERVICE_NAME)
.with_global_shutdown_timeout(Duration::from_secs(110))
.build();
let metrics_handle =
manager.register("metrics", ComponentOptions::new().is_observability(true));
let catalog_handle_lifecycle = manager.register(
"filter-catalog",
ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
);
let consumer_handle = manager.register(
"consumer",
ComponentOptions::new()
.with_graceful_shutdown(Duration::from_secs(30))
.with_liveness_deadline(Duration::from_secs(60))
.with_stall_threshold(3),
);
// Followers make their final commit after the consumer's worker drain. So each 45 s window
// outlasts the consumer's 30 s one, and the 10 s past `FOLLOWER_DRAIN_WAIT` (35 s) is for the
// commit itself.
let merge_follower_handle = manager.register(
"merge-follower",
ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(45)),
);
let transfer_follower_handle = manager.register(
"transfer-follower",
ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(45)),
);
// Registered only when the gate is on — a dormant deploy must not wait on a component that never
// starts.
let cascade_follower_handle = config.cohort_cascade_enabled.then(|| {
manager.register(
"cascade-follower",
ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(45)),
)
});
let seed_follower_handle = config.cohort_seed_consumer_enabled.then(|| {
manager.register(
"seed-follower",
ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(45)),
)
});
// Phase 1: the final checkpoint starts after every consumer and follower has made its final
// commit. Its positions are then those final commits, so a restore from it replays no event
// folded before the stop.
let checkpoint_handle = lineage.map(|_| {
manager.register(
"checkpoint",
ComponentOptions::new()
.with_graceful_shutdown(Duration::from_secs(60))
.with_shutdown_phase(1),
)
});
// Short graceful window: it holds no state and its next tick is disposable.
let tokio_monitor_handle = manager.register(
"tokio-runtime-monitor",
ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(2)),
);
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
let recorder_handle = if config.export_prometheus {
Some(observability::metrics::install_recorder())
} else {
None
};
let pool = get_pool_with_config(&config.database_url, config.pool_config())
.context("creating posthog_cohort database pool")?;
let (catalog, mut catalog_refresher) = CatalogHandle::with_refresher(
config.team_allowlist.clone(),
pool,
config.filter_catalog_refresh_interval(),
config.filter_catalog_refresh_jitter(),
config.cohort_cascade_enabled,
);
let boot_readiness = BootReadiness::new(catalog.clone());
// Bound before the restore, which can outlast the startup probe. `/_health` answers throughout;
// `/_ready` reads "boot recovery in progress" until the events consumer goes live.
let app = observability::health::router(
SERVICE_NAME,
readiness,
boot_readiness.clone(),
liveness,
recorder_handle,
);
let bind = config.bind_address();
let listener = TcpListener::bind(&bind)
.await
.with_context(|| format!("failed to bind observability server to {bind}"))?;
info!(address = %bind, "observability server listening");
let shutdown_signal = metrics_handle.shutdown_signal();
let health_server = tokio::spawn(async move {
axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal)
.await
});
match catalog_refresher.refresh().await {
Ok(stats) => info!(
teams = stats.teams,
unique_conditions = stats.unique_conditions,
"initial filter catalog loaded",
),
Err(err) => warn!(
error = %err,
"initial filter catalog load failed; catalog is empty until the refresh task succeeds",
),
}
let stream_consumer: StreamConsumer<CohortConsumerContext> = config
.consumer_client_config()
.create_with_context(CohortConsumerContext)
.context("creating cohort_stream_events consumer")?;
// Shared with the seed consumer's idle probe.
let stream_consumer = Arc::new(stream_consumer);
let merges_follower_consumer: Arc<StreamConsumer> = Arc::new(
config
.follower_client_config(&config.kafka_merge_consumer_group)
.create()
.context("creating person_merge_events follower consumer")?,
);
let transfers_follower_consumer: Arc<StreamConsumer> = Arc::new(
config
.follower_client_config(&config.kafka_merge_apply_consumer_group)
.create()
.context("creating cohort_merge_state_transfer follower consumer")?,
);
// Built only when the gate is on, so a dormant deploy needs no `cohort_cascade_events` topic.
let cascade_follower_consumer: Option<Arc<StreamConsumer>> = if config.cohort_cascade_enabled {
Some(Arc::new(
config
.follower_client_config(&config.kafka_cascade_consumer_group)
.create()
.context("creating cohort_cascade_events follower consumer")?,
))
} else {
None
};
// Gated: a gate-off deploy is safe without the seed topic existing at all.
let seed_follower_consumer: Option<Arc<StreamConsumer>> = if config.cohort_seed_consumer_enabled
{
Some(Arc::new(
config
.follower_client_config(&config.kafka_seed_consumer_group)
.create()
.context("creating cohort_stream_seed_events follower consumer")?,
))
} else {
None
};
let events_partitions =
fetch_partition_count(&stream_consumer, &config.cohort_stream_events_topic)?;
let merge_partitions =
fetch_partition_count(&merges_follower_consumer, &config.person_merge_events_topic)?;
let transfer_partitions = fetch_partition_count(
&transfers_follower_consumer,
&config.cohort_merge_state_transfer_topic,
)?;
// Checked before the store opens, because the store records the partition count it opens under.
// The merge math hashes `(team, person)` against `config.cohort_partition_count`, so the topics
// must not only be co-partitioned with each other but partitioned at exactly that count — a
// deploy/lane at N != the configured count silently misroutes every merge.
anyhow::ensure!(
events_partitions as u32 == config.cohort_partition_count,
"{} is partitioned at {} but COHORT_PARTITION_COUNT is {}: the merge partition arithmetic \
would misroute. Re-partition the topic to {} or set COHORT_PARTITION_COUNT to {}.",
config.cohort_stream_events_topic,
events_partitions,
config.cohort_partition_count,
config.cohort_partition_count,
events_partitions,
);
anyhow::ensure!(
merge_partitions == events_partitions && transfer_partitions == events_partitions,
"merge topics must be co-partitioned with {} ({} partitions): {} has {}, {} has {}",
config.cohort_stream_events_topic,
events_partitions,
config.person_merge_events_topic,
merge_partitions,
config.cohort_merge_state_transfer_topic,
transfer_partitions,
);
// A cascade for (team, person) must land on the partition owning that person's `cf_stage2` — the
// same partition number as the events topic — so refuse to start co-partitioned at a different
// count. Skipped when the gate is off (no cascade topic required).
if let Some(cascade_consumer) = &cascade_follower_consumer {
let cascade_partitions =
fetch_partition_count(cascade_consumer, &config.cohort_cascade_events_topic)?;
anyhow::ensure!(
cascade_partitions as u32 == config.cohort_partition_count,
"cohort_cascade_events must be co-partitioned with {} at COHORT_PARTITION_COUNT={}: {} has {}",
config.cohort_stream_events_topic,
config.cohort_partition_count,
config.cohort_cascade_events_topic,
cascade_partitions,
);
}
// A tile must land on the partition owning its person's state slice.
if let Some(seed_consumer) = &seed_follower_consumer {
let seed_partitions =
fetch_partition_count(seed_consumer, &config.cohort_stream_seed_events_topic)?;
anyhow::ensure!(
seed_partitions as u32 == config.cohort_partition_count,
"cohort_stream_seed_events must be co-partitioned with {} at COHORT_PARTITION_COUNT={}: {} has {}",
config.cohort_stream_events_topic,
config.cohort_partition_count,
config.cohort_stream_seed_events_topic,
seed_partitions,
);
// Nothing co-partitions with the marker topic, so only its existence matters. Failing here
// beats the alternative: a marker produce retrying forever while it holds the seed offset.
// Nested under the seed consumer on purpose: reconcile jobs are admitted only from seed
// tiles, so without it no marker can be produced and there is nothing to prove.
if config.cohort_seed_reconcile_enabled {
fetch_partition_count(seed_consumer, &config.cohort_reconcile_markers_topic).with_context(
|| {
format!(
"{} must exist before a processor with COHORT_SEED_RECONCILE_ENABLED starts. \
Provision the topic, or set COHORT_SEED_RECONCILE_ENABLED=false to start \
without the reconcile path.",
config.cohort_reconcile_markers_topic,
)
},
)?;
}
}
let groups = Arc::new(InputGroups::new(&config));
if lineage.is_some() {
ensure_one_filesystem(
Path::new(&config.checkpoint_local_dir),
Path::new(&config.store_path),
)
.context("checking the checkpoint directory")?;
}
let admitted = AdmittedSplit::admit(claim, groups.clone()).await;
let (store, restore) = open_store(&config, &admitted, lineage, groups.clone())
.await
.context("opening RocksDB state store")?;
let claimed = {
let groups = groups.clone();
tokio::task::spawn_blocking(move || admitted.claim(&groups))
.await
.context("the ownership claim panicked")?
.context("claiming this pod's partitions")?
};
let router = PartitionRouter::with_intake_cap(
config.partition_channel_buffer,
config.partition_intake_max_events,
config.seed_lane_cap(),
);
let offset_tracker = Arc::new(OffsetTracker::new());
let kafka_config = config.build_kafka_config();
let sink: Arc<dyn MembershipSink> = Arc::new(
KafkaMembershipSink::new(
&kafka_config,
config.cohort_membership_changed_topic.clone(),
)
.await
.context("creating membership producer")?,
);
// The transfer and marker sinks get the shorter `message.timeout.ms`: both produce inline on a
// partition worker and both are retried on a cadence — the transfer by a bounded inline loop, the
// marker by the drain sweeper — so a long per-attempt timeout repeats into a worker hold rather
// than costing it once. The membership and re-key sinks keep the shared 20 s: membership drops on
// fail (at-most-once) and the re-key produce rides the events-path offset gate
// (held-then-redelivered), so neither retries a black-holed produce on a timer.
let transfer_kafka_config = config.build_transfer_kafka_config();
let transfer_sink: Arc<dyn TransferSink> = Arc::new(
KafkaTransferSink::new(
&transfer_kafka_config,
config.cohort_merge_state_transfer_topic.clone(),
)
.await
.context("creating merge state transfer producer")?,
);
let stream_event_sink: Arc<dyn StreamEventSink> = Arc::new(
KafkaStreamEventSink::new(&kafka_config, config.cohort_stream_events_topic.clone())
.await
.context("creating straggler re-key producer")?,
);
let cascade_sink: Arc<dyn CascadeSink> = if config.cohort_cascade_enabled {
Arc::new(
KafkaCascadeSink::new(&kafka_config, config.cohort_cascade_events_topic.clone())
.await
.context("creating cohort_cascade_events producer")?,
)
} else {
Arc::new(NoopCascadeSink)
};
let seed_tile_sink: Arc<dyn SeedTileSink> = if config.cohort_seed_consumer_enabled {
Arc::new(
KafkaSeedTileSink::new(
&kafka_config,
config.cohort_stream_seed_events_topic.clone(),
)
.await
.context("creating cohort_stream_seed_events re-key producer")?,
)
} else {
Arc::new(NoopSeedTileSink)
};
let marker_sink: Arc<dyn ReconcileMarkerSink> = if config.cohort_seed_reconcile_enabled {
Arc::new(
KafkaReconcileMarkerSink::new(
&config.build_marker_kafka_config(),
config.cohort_reconcile_markers_topic.clone(),
)
.await
.context("creating cohort_reconcile_markers producer")?,
)
} else {
Arc::new(NoopReconcileMarkerSink)
};
let reconcile_backlog = Arc::new(ReconcileBacklog::default());
let merge_deps = Arc::new(MergeWorkerDeps {
transfer_sink,
stream_event_sink,
merge_tracker: Arc::new(OffsetTracker::new()),
transfer_tracker: Arc::new(OffsetTracker::new()),
retry: config.transfer_retry_policy(),
gc_scan_limit: config.merge_gc_scan_limit,
stage2_orphan_gc_enabled: config.stage2_orphan_gc_enabled,
cascade_sink,
cascade_tracker: Arc::new(OffsetTracker::new()),
cascade: config.cascade_config(),
partition_count: config.cohort_partition_count,
seed_tile_sink,
seed_tracker: Arc::new(OffsetTracker::new()),
// Unconditional (cheap): the event path observes regardless of the seed gate.
live_watermarks: Arc::new(LiveWatermarks::new()),
register_transfer_enabled: config.cohort_register_transfer_enabled,
reconcile: ReconcileDeps {
enabled: config.cohort_seed_reconcile_enabled,
scan_page: config.cohort_seed_reconcile_scan_page,
backlog: reconcile_backlog.clone(),
marker_sink,
},
person_seed: PersonSeedDeps {
enabled: config.cohort_seed_person_apply_enabled,
live_margin_ms: config.cohort_seed_person_live_margin_ms,
},
seed_budget: config.seed_run_budget(),
});
// The checkpoint sweeper keeps the raw `CohortStore` rather than the facade because of its
// must-not-panic policy (see checkpoint.rs).
let store_for_checkpoint = store.clone();
let handle = StoreHandle::new(store, config.offload_config());
let handle_for_stats = handle.clone();
let dispatcher = Arc::new(
EventDispatcher::new(
router,
offset_tracker,
handle,
catalog.clone(),
sink,
merge_deps,
)
.with_readiness(boot_readiness.clone()),
);
// Set once, before the consume loop and any worker spawn. The fsync-before-commit invariant is
// always on regardless — the gate only governs restore, not durability.
if config.durable_restore_enabled {
dispatcher.enable_durable_restore();
}
// Event-name fan-out gating, likewise set before any worker spawns.
dispatcher.set_event_name_gating(config.event_name_gating());
let mut followers = vec![
Follower::new(
merges_follower_consumer.clone(),
config.person_merge_events_topic.clone(),
),
Follower::new(
transfers_follower_consumer.clone(),
config.cohort_merge_state_transfer_topic.clone(),
),
];
if let Some(cascade_consumer) = &cascade_follower_consumer {
followers.push(Follower::new(
cascade_consumer.clone(),
config.cohort_cascade_events_topic.clone(),
));
}
if let Some(seed_consumer) = &seed_follower_consumer {
followers.push(Follower::new(
seed_consumer.clone(),
config.cohort_stream_seed_events_topic.clone(),
));
}
FollowerSet::new(followers)
.assign(&claimed)
.context("assigning the follower consumers this pod's partitions")?;
let guard = manager.monitor_background();
tokio::spawn(catalog_refresher.run(catalog_handle_lifecycle));
// Hold the first eviction pass past the boot window so its read burst doesn't stack on backlog
// catch-up; only the eviction sweep is delayed.
tokio::spawn(run_sweep_loop_delayed(
config.first_eviction_sweep_delay(),
DispatchSweeper::new(dispatcher.clone(), config.sweep_safety_margin_ms as i64),
config.sweep_interval(),
"eviction",
consumer_handle.shutdown_token(),
));
tokio::spawn(run_sweep_loop(
RedriveSweeper::new(dispatcher.clone()),
config.merge_redrive_interval(),
"redrive",
consumer_handle.shutdown_token(),
));
tokio::spawn(run_sweep_loop(
MergeGcSweeper::new(
dispatcher.clone(),
config.merge_marker_retention_ms as i64,
config.merge_tombstone_retention_ms as i64,
),
config.merge_gc_interval(),
"merge_gc",
consumer_handle.shutdown_token(),
));
if config.cohort_seed_reconcile_enabled {
tokio::spawn(run_sweep_loop(
ReconcileDrainSweeper::new(dispatcher.clone(), reconcile_backlog),
config.reconcile_tick_interval(),
"reconcile",
consumer_handle.shutdown_token(),
));
}
// Publish store cache/size metrics and the store filesystem's utilization via the sweep
// machinery, and Tokio runtime metrics via a separate monitor. The disk snapshot feeds the
// seed consumer's disk-backpressure gate; a sample surviving four sweep ticks unrefreshed
// means the sweep is wedged, so it expires rather than latching the gate.
let disk_state = Arc::new(SharedDiskUtilization::new(
config.stats_publish_interval() * 4,
));
tokio::spawn(run_sweep_loop(
StoreStatsSweeper::new(
handle_for_stats,
DiskProbe::new(PathBuf::from(&config.store_path), disk_state.clone()),
),
config.stats_publish_interval(),
"store_stats",
consumer_handle.shutdown_token(),
));
tokio::spawn(run_sweep_loop(
StoreFilesSweeper::new(
PathBuf::from(&config.store_path),
config
.checkpoint_enabled
.then(|| PathBuf::from(&config.checkpoint_local_dir)),
),
config.stats_publish_interval(),
"store_files",
consumer_handle.shutdown_token(),
));
tokio::spawn(
TokioRuntimeMonitor::new(
&tokio::runtime::Handle::current(),
config.stats_publish_interval(),
)
.start_monitoring(tokio_monitor_handle),
);
tokio::spawn(run_sweep_loop(
OwnershipAudit::new(groups.clone(), claimed.clone(), consumer_handle.clone()),
AUDIT_INTERVAL,
"ownership_audit",
consumer_handle.shutdown_token(),
));
// Whole-DB checkpoints to the local volume and S3, from boot to the final one on a graceful stop.
// The task waits for boot: a capture before the events rewind would record the broker's old
// offsets of a pending restore. Its first checkpoint then waits for the pod to be ready.
if let Some((checkpoint_handle, mut upload_requests)) = checkpoint_handle.zip(upload_requests) {
let sweeper = Arc::new(CheckpointSweeper::new(
store_for_checkpoint,
dispatcher.clone(),
groups,
claimed.clone(),
config.durability_config(),
upload_cadence(
config.checkpoint_interval_ms,
config.checkpoint_s3_upload_interval_ms,
),
consumer_handle.shutdown_token(),
));
let requests_stop = consumer_handle.shutdown_token();
let requested = sweeper.clone();
tokio::spawn(async move {
loop {
tokio::select! {
biased;
_ = requests_stop.cancelled() => break,
Some(()) = upload_requests.recv() => {
info!("checkpoint upload requested (SIGUSR1)");
requested.request_upload();
}
}
}
});
let stop = consumer_handle.shutdown_token();
let interval = config.checkpoint_interval();
let checkpoint_readiness = boot_readiness.clone();
tokio::spawn(async move {
tokio::select! {
biased;
// Boot never ended, so there are no settled positions to checkpoint.
_ = stop.cancelled() => checkpoint_handle.work_completed(),
_ = checkpoint_readiness.wait_until_dispatchable() => {
run_checkpoints(sweeper, &checkpoint_readiness, interval, checkpoint_handle)
.await;
}
}
});
}
let merge_follower = FollowerConsumer::<MergeRoute>::new(
merges_follower_consumer,
config.person_merge_events_topic.clone(),
claimed.stamp(),
dispatcher.clone(),
merge_follower_handle.clone(),
config.recv_batch_size,
config.recv_batch_timeout(),
config.offset_commit_interval(),
);
spawn_follower_after_boot(
boot_readiness.clone(),
merge_follower,
merge_follower_handle,
);
let transfer_follower = FollowerConsumer::<TransferRoute>::new(
transfers_follower_consumer,
config.cohort_merge_state_transfer_topic.clone(),
claimed.stamp(),
dispatcher.clone(),
transfer_follower_handle.clone(),
config.recv_batch_size,
config.recv_batch_timeout(),
config.offset_commit_interval(),
);
spawn_follower_after_boot(
boot_readiness.clone(),
transfer_follower,
transfer_follower_handle,
);
if let (Some(cascade_consumer), Some(cascade_handle)) =
(cascade_follower_consumer, cascade_follower_handle)
{
let cascade_follower = FollowerConsumer::<CascadeRoute>::new(
cascade_consumer,
config.cohort_cascade_events_topic.clone(),
claimed.stamp(),
dispatcher.clone(),
cascade_handle.clone(),
config.recv_batch_size,
config.recv_batch_timeout(),
config.offset_commit_interval(),
);
spawn_follower_after_boot(boot_readiness.clone(), cascade_follower, cascade_handle);
}
if let (Some(seed_consumer), Some(seed_handle)) = (seed_follower_consumer, seed_follower_handle)
{
let seed_topic = config.cohort_stream_seed_events_topic.clone();
let pauser: Arc<dyn PartitionPauser> = Arc::new(ConsumerPauser::new(
seed_consumer.clone(),
seed_topic.clone(),
));
let seed_follower = SeedFollowerConsumer::new(
seed_consumer,
seed_topic,
claimed.stamp(),
stream_consumer.clone(),
config.cohort_stream_events_topic.clone(),
dispatcher.clone(),
seed_handle.clone(),
pauser,
config.recv_batch_size,
config.recv_batch_timeout(),
config.offset_commit_interval(),
config.cohort_seed_fence_margin_ms,
config.seed_idle_probe_interval(),
config.seed_pacing_config()?,
disk_state.clone(),
);
let seed_readiness = boot_readiness.clone();
tokio::spawn(async move {
tokio::select! {
biased;
_ = seed_handle.shutdown_recv() => {}
_ = seed_readiness.wait_until_dispatchable() => seed_follower.process().await,
}
});
}
let events_consumer = CohortStreamEventsConsumer::new(
stream_consumer,
config.cohort_stream_events_topic.clone(),
dispatcher,
consumer_handle,
config.recv_batch_size,
config.recv_batch_timeout(),
config.offset_commit_interval(),
&claimed,
// A checkpoint restore boot finishes before anything folds; `None` when the store reopened
// or was created.
restore,
)
.context("assigning cohort_stream_events this pod's partitions")?;
tokio::spawn(events_consumer.process());
health_server
.await
.context("observability server task panicked")?
.context("observability server error")?;
metrics_handle.work_completed();
guard.wait().await?;
info!(service = SERVICE_NAME, "service stopped");
Ok(())
}
fn fetch_partition_count<C: ConsumerContext>(
consumer: &StreamConsumer<C>,
topic: &str,
) -> Result<usize> {
let metadata = consumer
.fetch_metadata(Some(topic), Duration::from_secs(10))
.with_context(|| format!("fetching broker metadata for {topic}"))?;
let topic_metadata = metadata
.topics()
.iter()
.find(|candidate| candidate.name() == topic)
.with_context(|| format!("topic {topic} missing from broker metadata"))?;
if let Some(err) = topic_metadata.error() {
anyhow::bail!("broker reports an error for topic {topic}: {err:?}");
}
let count = topic_metadata.partitions().len();
anyhow::ensure!(
count > 0,
"topic {topic} has no partitions in broker metadata"
);
Ok(count)
}
fn spawn_follower_after_boot<R: FollowerRoute>(
readiness: Arc<BootReadiness>,
follower: FollowerConsumer<R>,
handle: Handle,
) {
tokio::spawn(async move {
tokio::select! {
biased;
_ = handle.shutdown_recv() => {}
_ = readiness.wait_until_dispatchable() => follower.process().await,
}
});
}
/// Log a redacted startup summary. Deliberately omits `database_url` (carries credentials).
fn log_startup(config: &Config) {
info!(
service = SERVICE_NAME,
bind_address = %config.bind_address(),
kafka_hosts = %config.kafka_hosts,
input_topic = %config.cohort_stream_events_topic,
output_topic = %config.cohort_membership_changed_topic,
reconcile_markers_topic = %config.cohort_reconcile_markers_topic,
consumer_group = %config.kafka_consumer_group,
merge_topic = %config.person_merge_events_topic,
transfer_topic = %config.cohort_merge_state_transfer_topic,
merge_consumer_group = %config.kafka_merge_consumer_group,
merge_apply_consumer_group = %config.kafka_merge_apply_consumer_group,
pod_identity = config.pod_identity().unwrap_or("<none>"),
pod_count = config.cohort_pod_count,
reshard_from_pod_count = ?config.cohort_reshard_from_pod_count,
recv_batch_size = config.recv_batch_size,
partition_channel_buffer = config.partition_channel_buffer,
store_path = %config.store_path,
wipe_store_on_start = config.wipe_store_on_start,
durable_restore_enabled = config.durable_restore_enabled,
filter_catalog_refresh_secs = config.filter_catalog_refresh_secs,
filter_catalog_refresh_jitter_secs = config.filter_catalog_refresh_jitter_secs,
team_allowlist = ?config.team_allowlist,
cohort_cascade_enabled = config.cohort_cascade_enabled,
cascade_topic = %config.cohort_cascade_events_topic,
cascade_consumer_group = %config.kafka_cascade_consumer_group,
cohort_seed_consumer_enabled = config.cohort_seed_consumer_enabled,
seed_topic = %config.cohort_stream_seed_events_topic,
seed_consumer_group = %config.kafka_seed_consumer_group,
seed_fence_margin_ms = config.cohort_seed_fence_margin_ms,
cohort_seed_live_lag_pause_ms = config.cohort_seed_live_lag_pause_ms,
cohort_seed_live_lag_resume_ms = config.cohort_seed_live_lag_resume_ms,
cohort_seed_disk_pause_pct = config.cohort_seed_disk_pause_pct,
cohort_seed_disk_resume_pct = config.cohort_seed_disk_resume_pct,
cohort_seed_reconcile_enabled = config.cohort_seed_reconcile_enabled,
cohort_seed_reconcile_scan_page = config.cohort_seed_reconcile_scan_page,
cohort_seed_reconcile_tick_interval_ms = config.cohort_seed_reconcile_tick_interval_ms,
"starting cohort-stream-processor",
);
}
/// JSON structured logging in production; human-readable when `RUST_LOG` requests debug.
fn init_tracing() {
let is_debug = std::env::var("RUST_LOG")
.map(|v| v.contains("debug"))
.unwrap_or(false);
let log_layer = if is_debug {
fmt::layer()
.with_target(true)
.with_level(true)
.with_ansi(true)
.with_filter(
EnvFilter::builder()
.with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
.from_env_lossy(),
)
.boxed()
} else {
fmt::layer()
.json()
.flatten_event(true)
.with_current_span(true)
.with_filter(
EnvFilter::builder()
.with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
.from_env_lossy(),
)
.boxed()
};
let subscriber = tracing_subscriber::registry().with(log_layer);
// Without `hotpath`, this returns an unfiltered no-op layer. That layer lifts the max level to
// TRACE, so every DEBUG/TRACE call site builds its event, sqlx's per-query log included.
#[cfg(feature = "hotpath")]
let subscriber = subscriber.with(hotpath::sqlx_tracing_layer());
subscriber.init();
}