:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::{sync::Arc, time::Duration};
use app_context::AppContext;
use batch_ingestion::process_batch;
use common_kafka::kafka_consumer::{RecvErr, SingleTopicConsumer};
use config::Config;
use metrics_consts::{
BATCH_ACQUIRE_TIME, CACHE_FILL_RATIO, CACHE_LEN, COMPACTED_UPDATES, DUPLICATES_IN_BATCH,
EMPTY_EVENTS, EVENTS_RECEIVED, EVENT_PARSE_ERROR, FORCED_SMALL_BATCH, OFFSET_STORE_FAILURES,
RECV_DEQUEUED, SKIPPED_DUE_TO_TEAM_FILTER, UPDATES_FILTERED_BY_CACHE, UPDATES_PER_EVENT,
UPDATES_SEEN, WORKER_BLOCKED,
};
use types::{Event, Update};
use ahash::AHashSet;
use tokio::sync::mpsc::error::TrySendError;
use tracing::{error, info, warn};
use update_cache::Cache;
use crate::{
measuring_channel::{MeasuringReceiver, MeasuringSender},
metrics_consts::CHANNEL_MESSAGES_IN_FLIGHT,
};
pub mod api;
pub mod app_context;
pub mod batch_ingestion;
pub mod config;
pub mod group_type_resolver;
pub mod measuring_channel;
pub mod metrics_buckets;
pub mod metrics_consts;
pub mod read_filter;
pub mod types;
pub mod update_cache;
pub async fn update_consumer_loop(
config: Config,
cache: Arc<Cache>,
context: Arc<AppContext>,
mut channel: MeasuringReceiver<Update>,
handle: lifecycle::Handle,
) {
let _guard = handle.process_scope();
loop {
let mut batch = Vec::with_capacity(config.update_batch_size);
let batch_start = tokio::time::Instant::now();
let batch_time = common_metrics::timing_guard(BATCH_ACQUIRE_TIME, &[]);
while batch.len() < config.update_batch_size {
handle.report_healthy();
metrics::gauge!(CHANNEL_MESSAGES_IN_FLIGHT)
.set(channel.get_inflight_messages_count() as f64);
let remaining_capacity = config.update_batch_size - batch.len();
// We race these two, so we can escape this loop and do a small batch if we've been waiting too long
let recv = channel.recv_many(&mut batch, remaining_capacity);
let sleep = tokio::time::sleep(Duration::from_secs(1));
tokio::select! {
_ = handle.shutdown_recv() => {
info!("Consumer loop received shutdown signal");
return;
}
got = recv => {
if got == 0 {
info!("Channel closed, all producers exited");
return;
}
metrics::gauge!(RECV_DEQUEUED).set(got as f64);
continue;
}
_ = sleep => {
if batch_start.elapsed() > Duration::from_secs(config.max_issue_period) {
warn!("Forcing small batch due to time limit");
metrics::counter!(FORCED_SMALL_BATCH).increment(1);
break;
}
}
}
}
batch_time.fin();
if batch.is_empty() {
continue;
}
// We de-duplicate the batch, in case racing inserts slipped through the shared-cache filter. This
// is important because duplicate updates touch the same row, and we issue in parallel, so we'd end
// up deadlocking ourselves. We can still encounter deadlocks due to other pods, but those should
// be rarer, and we use retries to handle them.
let start_len = batch.len();
batch.sort_unstable();
batch.dedup();
metrics::counter!(DUPLICATES_IN_BATCH).increment((start_len - batch.len()) as u64);
// Per-subcache size gauges once per batch. Hit/miss/eviction counters
// are emitted from inside `Cache::contains_key` and the EvictingLifecycle
// impl in `update_cache.rs`, not here.
let per_cache = [
(
config.eventdefs_cache_capacity,
"eventdefs",
cache.eventdefs_len(),
),
(
config.eventprops_cache_capacity,
"eventprops",
cache.eventprops_len(),
),
(
config.propdefs_cache_capacity,
"propdefs",
cache.propdefs_len(),
),
];
for (cap, label, len) in per_cache {
let cap_f = cap as f64;
metrics::gauge!(CACHE_FILL_RATIO, &[("cache", label)]).set(if cap_f > 0.0 {
len as f64 / cap_f
} else {
0.0
});
metrics::gauge!(CACHE_LEN, &[("cache", label)]).set(len as f64);
}
// enrich batch group events with resolved group_type_indices
// before passing along to process_batch
let _unused = context
.resolve_group_types_indexes(&mut batch)
.await
.map_err(|e| {
warn!(
"Failed resolving group type indices for batch, got: {:?}",
e
)
});
handle.report_healthy();
process_batch(
&config,
cache.clone(),
&context.pool,
context.read_pool.clone(),
batch,
&handle,
)
.await;
}
}
pub async fn update_producer_loop(
config: Config,
consumer: SingleTopicConsumer,
shared_cache: Arc<Cache>,
channel: MeasuringSender<Update>,
handle: lifecycle::Handle,
) {
let _guard = handle.process_scope();
let mut batch = AHashSet::with_capacity(config.compaction_batch_size);
let mut last_send = tokio::time::Instant::now();
let drain_interval = Duration::from_secs(config.producer_drain_interval_secs);
// These fire per event or per update, so resolve the handles once instead of
// paying a registry lookup (hash + CAS + handle drop) on every increment.
// Safe only because main installs the metrics recorder before spawning us.
let events_received = metrics::counter!(EVENTS_RECEIVED);
let updates_seen = metrics::counter!(UPDATES_SEEN);
let updates_per_event = metrics::histogram!(UPDATES_PER_EVENT);
let compacted_updates = metrics::counter!(COMPACTED_UPDATES);
let updates_filtered_by_cache = metrics::counter!(UPDATES_FILTERED_BY_CACHE);
let skipped_due_to_team_filter = metrics::counter!(SKIPPED_DUE_TO_TEAM_FILTER);
loop {
// The timer arm is what lets the flush check at the bottom of the loop run while no
// events are arriving. Without it the loop parks in json_recv(), so a partial compaction
// batch left behind when a partition goes quiet stays unwritten until traffic resumes.
// The tick period is deliberately independent of drain_interval; it only bounds how
// often the elapsed check is re-evaluated, not how long a batch may sit.
let recv_result = tokio::select! {
_ = handle.shutdown_recv() => {
info!("Producer loop shutting down");
return;
}
r = consumer.json_recv() => Some(r),
_ = tokio::time::sleep(Duration::from_secs(1)) => None,
};
if let Some(recv_result) = recv_result {
let (event, offset): (Event, _) = match recv_result {
Ok(r) => r,
Err(RecvErr::Empty) => {
warn!("Received empty event");
metrics::counter!(EMPTY_EVENTS).increment(1);
continue;
}
Err(RecvErr::Serde(e)) => {
metrics::counter!(EVENT_PARSE_ERROR).increment(1);
warn!("Failed to parse event: {:?}", e);
continue;
}
Err(RecvErr::Kafka(e)) => {
handle.signal_failure(format!("Kafka error: {e:?}"));
return;
}
};
// NOTE: we extended the autocommit interval in production envs to 20 seconds
// as a temporary remediation for events already buffered in a pod's internal
// queue being skipped when the consumer group rebalances or the service is
// redeployed. Long-term fix: start tracking max partition offsets in the
// Update batches and commit them only when each batch succeeds. This will
// not be perfect, as batch writes are async and can complete out of order,
// but is better than what we're doing right now
let curr_offset = offset.get_value();
match offset.store() {
Ok(_) => (),
Err(e) => {
metrics::counter!(OFFSET_STORE_FAILURES).increment(1);
// TODO: consumer json_recv() should expose the source partition ID too
error!("update_producer_loop: failed to store offset {curr_offset}, got: {e}");
}
}
if !config
.filter_mode
.should_process(&config.filtered_teams.teams, event.team_id)
{
skipped_due_to_team_filter.increment(1);
continue;
}
let updates = event.into_updates_with(
config.update_count_skip_threshold,
config.eventdef_last_seen_floor_secs,
);
events_received.increment(1);
updates_seen.increment(updates.len() as u64);
updates_per_event.record(updates.len() as f64);
for update in updates {
// insert returns false on duplicates, so one hash covers the
// membership check and the insert.
if !batch.insert(update) {
compacted_updates.increment(1);
}
}
}
// We do the full batch insert before checking the time/batch size, because if we did this
// inside the for update in updates loop, under extremely low-load situations, we'd push a
// single update into the channel, then push the rest into the batch, and loop around to
// wait on the next event, which might come an arbitrary amount of time later. This bit me
// in testing, and while it's not a correctness problem and under normal load we'd never
// see it, we may as well just do the full batch insert first.
if batch.len() >= config.compaction_batch_size || last_send.elapsed() > drain_interval {
last_send = tokio::time::Instant::now();
for update in batch.drain() {
if shared_cache.contains_key(&update) {
// kept for back-compat; equivalent to sum(prop_defs_cache_hits)
updates_filtered_by_cache.increment(1);
continue;
}
// Optimistic caching: entries go in before they are persisted, and are evicted
// again if the batch write fails. That trade is acceptable because batch writes
// rarely fail, and the alternative costs a second pass over every batch.
shared_cache.insert(update.clone());
match channel.try_send(update) {
Ok(_) => {}
Err(TrySendError::Full(update)) => {
warn!("Worker blocked");
metrics::counter!(WORKER_BLOCKED).increment(1);
if channel.send(update).await.is_err() {
return;
}
}
Err(e) => {
warn!("Coordinator send failed: {:?}", e);
return;
}
}
}
}
}
}