:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! App-side intake backpressure: a per-partition ceiling on events resident in a worker's live
//! lane. Over the cap, new messages are refused (→ held → paused), never buffered. Seeds are not
//! metered here — their lane holds one seed per slot, so its capacity is their cap.
//!
//! The counter cannot drift: [`PartitionIntake::try_admit`] reserves, [`MeteredReceiver`] releases
//! on the next `recv` and on `Drop`, and the router releases eagerly on a failed send. Both sides
//! share [`ShuffleMessage::counts_toward_intake`], so they can never disagree on what counts.
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use metrics::gauge;
use tokio::sync::mpsc;
use super::shuffle_message::ShuffleMessage;
use crate::observability::metrics::PARTITION_INTAKE_EVENTS;
/// Intake-counted events in a batch; maintenance ticks count 0.
pub fn count_intake(batch: &[ShuffleMessage]) -> usize {
batch
.iter()
.filter(|message| message.counts_toward_intake())
.count()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Admission {
/// Reserved under the cap; the caller must send the batch or release the reservation.
Admitted,
/// At the event ceiling; the caller holds the batch and pauses the partition.
Rejected,
}
/// Per-partition ceiling on un-drained events in the live lane.
///
/// Releases (from the worker's [`MeteredReceiver`] and, on a failed send, the router) only ever lower
/// the count, so [`try_admit`](Self::try_admit)'s plain load-then-add never overshoots `cap`: a
/// concurrent release between its load and its add can only have freed room.
pub struct PartitionIntake {
cap: usize,
in_flight_events: AtomicUsize,
partition_label: Arc<str>,
}
impl PartitionIntake {
pub fn new(partition: i32, cap: usize) -> Self {
Self {
cap,
in_flight_events: AtomicUsize::new(0),
partition_label: Arc::from(partition.to_string()),
}
}
/// Reserve `count` events if they fit under `cap`. An idle partition always admits — even an
/// over-cap batch — so `cap` is a soft ceiling on the steady state, not a hard per-batch limit.
///
/// Not a CAS: correct only because a single task (the consume loop) admits for a given partition.
/// Two concurrent admitters could both pass the check and overshoot `cap`.
pub fn try_admit(&self, count: usize) -> Admission {
let current = self.in_flight_events.load(Ordering::Acquire);
if current == 0 || current + count <= self.cap {
self.in_flight_events.fetch_add(count, Ordering::AcqRel);
// Sample fresh: a concurrent release may have lowered the counter below the reserved total.
self.set_gauge(self.in_flight_events.load(Ordering::Acquire));
Admission::Admitted
} else {
Admission::Rejected
}
}
/// Release a reservation once its events drain (or fail to land). Saturating, so an accounting
/// slip can't underflow into a permanent stall.
pub fn release(&self, count: usize) {
if count == 0 {
return;
}
let mut current = self.in_flight_events.load(Ordering::Acquire);
loop {
let next = current.saturating_sub(count);
match self.in_flight_events.compare_exchange_weak(
current,
next,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => {
self.set_gauge(next);
return;
}
Err(observed) => current = observed,
}
}
}
pub fn in_flight(&self) -> usize {
self.in_flight_events.load(Ordering::Acquire)
}
fn set_gauge(&self, value: usize) {
gauge!(PARTITION_INTAKE_EVENTS, "partition" => self.partition_label.clone())
.set(value as f64);
}
}
/// Worker-side receiver that releases each batch's event reservation back to the partition's
/// [`PartitionIntake`]. Releases on the *next* `recv` and on `Drop`, so the budget also covers the
/// batch the worker currently holds.
pub struct MeteredReceiver {
receiver: mpsc::Receiver<Vec<ShuffleMessage>>,
intake: Arc<PartitionIntake>,
/// Events reserved for the last batch handed out, released on the next `recv`/`Drop`.
outstanding: usize,
}
impl MeteredReceiver {
pub fn new(
receiver: mpsc::Receiver<Vec<ShuffleMessage>>,
intake: Arc<PartitionIntake>,
) -> Self {
Self {
receiver,
intake,
outstanding: 0,
}
}
/// Uncapped intake wrapper for tests: only the mpsc slots bound.
pub fn unmetered(receiver: mpsc::Receiver<Vec<ShuffleMessage>>) -> Self {
Self::new(receiver, Arc::new(PartitionIntake::new(0, usize::MAX)))
}
/// Release the previous batch, then await the next.
///
/// Cancel-safe, so the worker can poll it as a `select!` branch: the release runs on the first
/// poll, strictly after the previous batch was processed, and is idempotent when the future is
/// dropped and re-created on the next round.
pub async fn recv(&mut self) -> Option<Vec<ShuffleMessage>> {
self.release_outstanding();
let batch = self.receiver.recv().await?;
self.outstanding = count_intake(&batch);
Some(batch)
}
fn release_outstanding(&mut self) {
if self.outstanding > 0 {
self.intake.release(self.outstanding);
self.outstanding = 0;
}
}
/// Non-awaiting drain for tests; releases like [`recv`](Self::recv).
#[cfg(test)]
pub fn try_recv(&mut self) -> Result<Vec<ShuffleMessage>, mpsc::error::TryRecvError> {
self.release_outstanding();
let batch = self.receiver.try_recv()?;
self.outstanding = count_intake(&batch);
Ok(batch)
}
}
impl Drop for MeteredReceiver {
fn drop(&mut self) {
self.release_outstanding();
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::consumers::events::CohortStreamEvent;
fn event() -> ShuffleMessage {
ShuffleMessage::Event {
event: Box::new(CohortStreamEvent {
team_id: 1,
person_id: "p".to_string(),
distinct_id: "d".to_string(),
uuid: "u".to_string(),
event: "$pageview".to_string(),
timestamp: "2026-05-26 12:34:56.789000".to_string(),
properties: None,
person_properties: None,
elements_chain: None,
source_offset: 0,
source_partition: 0,
redirected_from: None,
redirect_hops: 0,
}),
cse_offset: 0,
broker_ts_ms: None,
}
}
fn events(n: usize) -> Vec<ShuffleMessage> {
(0..n).map(|_| event()).collect()
}
#[test]
fn count_intake_counts_events_but_ignores_maintenance() {
let batch = vec![
event(),
ShuffleMessage::Sweep { due_before_ms: 1 },
event(),
ShuffleMessage::RedrivePendingTransfers,
];
assert_eq!(count_intake(&batch), 2);
}
#[test]
fn admits_under_cap_and_rejects_over_cap() {
let intake = PartitionIntake::new(0, 10);
assert_eq!(intake.try_admit(6), Admission::Admitted);
assert_eq!(intake.in_flight(), 6);
assert_eq!(intake.try_admit(5), Admission::Rejected);
assert_eq!(intake.in_flight(), 6);
assert_eq!(intake.try_admit(4), Admission::Admitted);
assert_eq!(intake.in_flight(), 10);
}
#[test]
fn an_idle_partition_admits_an_over_cap_batch_for_progress() {
let intake = PartitionIntake::new(0, 10);
assert_eq!(intake.try_admit(25), Admission::Admitted);
assert_eq!(intake.in_flight(), 25);
assert_eq!(intake.try_admit(1), Admission::Rejected);
}
#[test]
fn release_is_saturating() {
let intake = PartitionIntake::new(0, 10);
intake.try_admit(3);
intake.release(10);
assert_eq!(intake.in_flight(), 0);
}
#[tokio::test]
async fn metered_receiver_releases_the_previous_batch_on_the_next_recv() {
let intake = Arc::new(PartitionIntake::new(0, 100));
let (tx, rx) = mpsc::channel(8);
let mut rx = MeteredReceiver::new(rx, intake.clone());
assert_eq!(intake.try_admit(3), Admission::Admitted);
tx.send(events(3)).await.unwrap();
assert_eq!(intake.try_admit(2), Admission::Admitted);
tx.send(events(2)).await.unwrap();
assert_eq!(intake.in_flight(), 5);
assert_eq!(count_intake(&rx.recv().await.unwrap()), 3);
assert_eq!(intake.in_flight(), 5, "the in-hand batch stays counted");
assert_eq!(count_intake(&rx.recv().await.unwrap()), 2);
assert_eq!(intake.in_flight(), 2, "recv released the previous batch");
drop(rx);
assert_eq!(
intake.in_flight(),
0,
"drop releases the last in-hand batch"
);
}
#[tokio::test]
async fn a_maintenance_batch_does_not_move_the_counter() {
let intake = Arc::new(PartitionIntake::new(0, 100));
let (tx, rx) = mpsc::channel(8);
let mut rx = MeteredReceiver::new(rx, intake.clone());
tx.send(vec![ShuffleMessage::Sweep { due_before_ms: 1 }])
.await
.unwrap();
rx.recv().await.unwrap();
drop(rx);
assert_eq!(intake.in_flight(), 0);
}
}