:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use lifecycle::Handle;
use metrics::{counter, gauge, histogram};
use personhog_proto::personhog::types::v1::Person;
use prost::Message;
use rdkafka::error::KafkaError;
use rdkafka::message::Message as KafkaMessage;
use tokio::sync::mpsc;
use tracing::{error, info, warn};
use crate::buffer::{BufferedPerson, PersonBuffer};
use crate::kafka::{PersonConsumer, Source};
/// Batch of persons and their Kafka offsets, sent from consumer to writer.
pub struct FlushBatch {
pub persons: Vec<BufferedPerson>,
pub offsets: HashMap<Source, i64>,
/// Timestamp of the oldest Kafka message in this batch (millis since epoch).
/// Used to compute end-to-end latency from ingestion to PG commit.
pub oldest_message_ts_ms: Option<i64>,
}
/// Their offsets were never committed, so the new owner re-reads these rows;
/// writing them here as well would only race that owner.
fn drop_revoked_partitions(consumer: &PersonConsumer, lanes: &mut [Lane]) {
let revoked = consumer.take_revoked();
if revoked.is_empty() {
return;
}
let dropped: usize = lanes
.iter_mut()
.map(|lane| lane.buffer.remove_sources(&revoked))
.sum();
counter!("personhog_writer_partitions_revoked_total").increment(revoked.len() as u64);
counter!("personhog_writer_revoked_rows_dropped_total", "stage" => "buffer")
.increment(dropped as u64);
info!(
partitions = revoked.len(),
rows = dropped,
"dropped buffered rows for revoked partitions"
);
}
/// One writer lane: a dedup buffer plus the channel to its writer task.
/// Partitions map to lanes by `partition % lanes`, so a partition's
/// messages always flow through the same lane — Kafka's cumulative
/// per-partition commits stay safe because each lane commits only the
/// partitions it owns, in order.
struct Lane {
buffer: PersonBuffer,
flush_tx: mpsc::Sender<FlushBatch>,
}
/// Reads from Kafka, decodes Person protos, buffers with dedup per lane,
/// and sends batches to the lane writer tasks for PG upsert + offset commit.
pub struct ConsumerTask {
consumer: Arc<PersonConsumer>,
lanes: Vec<Lane>,
flush_interval: Duration,
flush_buffer_size: usize,
handle: Handle,
/// The last message's topic, shared by every row from it.
topic: Arc<str>,
}
impl ConsumerTask {
/// `lane_txs` carries one sender per writer lane; each lane gets its own
/// dedup buffer of `per_lane_capacity` entries. A single-element vec
/// reproduces the single-writer pipeline.
pub fn new(
consumer: Arc<PersonConsumer>,
lane_txs: Vec<mpsc::Sender<FlushBatch>>,
per_lane_capacity: usize,
flush_interval: Duration,
flush_buffer_size: usize,
handle: Handle,
) -> Self {
assert!(!lane_txs.is_empty(), "at least one writer lane is required");
// The nonblocking size flush must fire while the lane still has
// headroom below its hard cap; otherwise every flush degrades to the
// blocking backpressure path and one busy writer stalls consumption
// for all lanes again.
let max_flush_size = (per_lane_capacity / 2).max(1);
let flush_buffer_size = if flush_buffer_size > max_flush_size {
warn!(
configured = flush_buffer_size,
clamped = max_flush_size,
per_lane_capacity,
"flush_buffer_size clamped to half the per-lane capacity"
);
max_flush_size
} else {
flush_buffer_size
};
let lanes = lane_txs
.into_iter()
.map(|flush_tx| Lane {
buffer: PersonBuffer::new(per_lane_capacity),
flush_tx,
})
.collect();
let topic = Arc::from(consumer.topic().as_str());
Self {
consumer,
lanes,
flush_interval,
flush_buffer_size,
handle,
topic,
}
}
}
/// Rows share one allocation per topic; the cache is the last topic seen.
fn source_from(cache: &mut Arc<str>, topic: &str, partition: i32) -> Source {
if &**cache != topic {
*cache = Arc::from(topic);
}
Source {
topic: Arc::clone(cache),
partition,
}
}
impl ConsumerTask {
pub async fn run(mut self) {
let _guard = self.handle.process_scope();
let mut flush_timer = tokio::time::interval(self.flush_interval);
info!(lanes = self.lanes.len(), "Consumer task starting");
loop {
// Backpressure: a full lane drains one capped batch before the
// loop re-checks, so even the hard-cap path releases its backlog
// as bounded batches rather than one giant flush.
if let Some(idx) = self.lanes.iter().position(|l| l.buffer.is_full()) {
counter!("personhog_writer_flushes_by_trigger_total", "trigger" => "backpressure")
.increment(1);
self.send_flush(idx).await;
continue;
}
tokio::select! {
biased;
_ = self.handle.shutdown_recv() => {
info!("Shutdown signal, flushing remaining buffers");
for idx in 0..self.lanes.len() {
while !self.lanes[idx].buffer.is_empty() {
counter!("personhog_writer_flushes_by_trigger_total", "trigger" => "shutdown")
.increment(1);
self.send_flush(idx).await;
}
}
break;
}
_ = flush_timer.tick() => {
// Drain each lane fully, one capped batch at a time. The
// blocking send paces the drain at the writer's speed, so
// a backlogged lane empties as a sequence of bounded
// batches instead of one giant one.
for idx in 0..self.lanes.len() {
while !self.lanes[idx].buffer.is_empty() {
counter!("personhog_writer_flushes_by_trigger_total", "trigger" => "timer")
.increment(1);
self.send_flush(idx).await;
}
}
}
msg = self.consumer.recv() => {
match msg {
Ok(borrowed_msg) => {
let partition = borrowed_msg.partition();
let offset = borrowed_msg.offset();
let ts_ms = borrowed_msg.timestamp().to_millis();
let source =
source_from(&mut self.topic, borrowed_msg.topic(), partition);
drop_revoked_partitions(&self.consumer, &mut self.lanes);
if let Some(payload) = borrowed_msg.payload() {
match Person::decode(payload) {
Ok(person) => {
counter!("personhog_writer_messages_consumed_total")
.increment(1);
let idx = (partition as usize) % self.lanes.len();
let lane = &mut self.lanes[idx];
lane.buffer.insert(person, source, offset, ts_ms);
// Size-triggered flushes never block the
// consume loop: if this lane's writer is
// still busy, keep buffering and reading —
// other lanes' writers stay fed. The lane's
// hard capacity above is the real limit.
if lane.buffer.len() >= self.flush_buffer_size
&& self.try_send_flush(idx)
{
counter!(
"personhog_writer_flushes_by_trigger_total",
"trigger" => "size"
)
.increment(1);
}
}
Err(e) => {
counter!("personhog_writer_decode_errors_total")
.increment(1);
warn!(
error = %e, partition, offset,
"failed to decode Person proto"
);
}
}
}
}
Err(KafkaError::PartitionEOF(partition)) => {
if self.consumer.is_draining() {
self.flush_partition_end(partition).await;
}
}
Err(e) => {
counter!("personhog_writer_kafka_errors_total").increment(1);
warn!(error = %e, "Kafka recv error");
// A non-fatal error is librdkafka riding out a
// blip and needs nothing from us. A fatal
// client state never recovers: left here, the
// loop re-polls a dead client forever while
// the pod reports healthy and its partitions'
// records go unwritten.
if let Some((code, reason)) = self.consumer.fatal_error() {
self.handle.signal_failure(format!(
"Kafka client entered a fatal state ({code:?}): {reason}"
));
return;
}
}
}
}
}
}
info!("Consumer task stopped");
}
/// A sealed partition's tail flushes at once instead of waiting out the timer.
async fn flush_partition_end(&mut self, partition: i32) {
let idx = (partition as usize) % self.lanes.len();
while !self.lanes[idx].buffer.is_empty() {
counter!("personhog_writer_flushes_by_trigger_total", "trigger" => "partition_end")
.increment(1);
self.send_flush(idx).await;
}
counter!("personhog_writer_partition_ends_total").increment(1);
info!(partition, "reached the end of a sealed partition");
}
/// Drain up to `flush_buffer_size` rows (whole partitions only) from a
/// lane's buffer into a batch. Offset and buffer-size gauges are
/// recorded here, per flush — updating them per message is too
/// expensive for the consume loop (dynamic-label registry lookups).
fn take_batch(&mut self, idx: usize) -> Option<FlushBatch> {
drop_revoked_partitions(&self.consumer, &mut self.lanes);
let lane = &mut self.lanes[idx];
let batch = lane.buffer.drain_up_to(self.flush_buffer_size)?;
for (source, offset) in &batch.offsets {
gauge!(
"personhog_writer_partition_offset",
"partition" => source.partition.to_string()
)
.set(*offset as f64);
}
let total: usize = self.lanes.iter().map(|l| l.buffer.len()).sum();
gauge!("personhog_writer_buffer_size").set(total as f64);
Some(FlushBatch {
persons: batch.persons,
offsets: batch.offsets,
oldest_message_ts_ms: batch.oldest_message_ts_ms,
})
}
/// Drain one capped batch from a lane and send it to the writer task,
/// waiting until the writer accepts it. Used on the timer, shutdown, and
/// hard-capacity paths, where blocking the consume loop is intended
/// backpressure.
async fn send_flush(&mut self, idx: usize) {
let Some(batch) = self.take_batch(idx) else {
return;
};
let count = batch.persons.len();
let start = std::time::Instant::now();
if self.lanes[idx].flush_tx.send(batch).await.is_err() {
error!(rows = count, lane = idx, "writer task gone, dropping batch");
self.handle
.signal_failure(format!("Writer task channel closed (lane {idx})"));
}
histogram!("personhog_writer_channel_send_duration_ms")
.record(start.elapsed().as_secs_f64() * 1000.0);
}
/// Flush a lane only if its writer channel has capacity right now.
/// Returns whether a batch was handed off. Never awaits, so a busy
/// writer on one lane can't stall consumption for the others.
fn try_send_flush(&mut self, idx: usize) -> bool {
let tx = self.lanes[idx].flush_tx.clone();
// Bound to a local first: the permit borrows `tx`, so a tail-expression
// match would outlive it.
let flushed = match tx.try_reserve() {
Ok(permit) => {
if let Some(batch) = self.take_batch(idx) {
permit.send(batch);
}
true
}
Err(mpsc::error::TrySendError::Full(())) => false,
Err(mpsc::error::TrySendError::Closed(())) => {
error!(lane = idx, "writer task gone, cannot flush");
self.handle
.signal_failure(format!("Writer task channel closed (lane {idx})"));
true
}
};
flushed
}
}