:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ingestion-consumer): add the dispatch governor and its RTT average (#121088)
José Sequeira698130232e8b
diff --git a/rust/ingestion-consumer/src/batcher/governor.rs b/rust/ingestion-consumer/src/batcher/governor.rsnew file mode 100644index 000000000000..49a33b4d72df--- /dev/null+++ b/rust/ingestion-consumer/src/batcher/governor.rs@@ -0,0 +1,447 @@+//! Governs how many requests the batcher keeps in flight from request+//! round-trip time.+//!+//! The permit bounds follow the routable workers: the ceiling is the+//! per-worker cap times the pool, so at the ceiling the permits are the free+//! worker slots. Permits move by one at most once per step interval, because+//! a step per settlement walks the pool to the floor in one round trip and+//! then sawtooths against the average. With adaptation off the pool is a+//! shadow: it moves by the law and reports what it would have done, while the+//! batcher sizes its passes by free slots as before.++use std::time::{Duration, Instant};++const RTT_TIME_CONSTANT: Duration = Duration::from_secs(10);++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]+pub enum Adaptation {+ #[default]+ Off,+ On,+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum Direction {+ Grow,+ Shrink,+}++/// One step of the law that moved the pool. `applied` is false when+/// adaptation is off, so the step moved only the shadow pool.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub struct Decision {+ pub direction: Direction,+ pub applied: bool,+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub struct GovernorSettings {+ pub adaptation: Adaptation,+ /// An RTT average above it shrinks the pool. `None` never shrinks.+ pub latency_budget: Option<Duration>,+ pub min_permits_per_worker: usize,+ pub step_interval: Duration,+}++impl GovernorSettings {+ /// Measures and reports with no budget, so the pool stays at the ceiling.+ pub fn observing() -> Self {+ Self {+ adaptation: Adaptation::Off,+ latency_budget: None,+ min_permits_per_worker: 1,+ step_interval: Duration::from_secs(1),+ }+ }+}++/// A point-in-time view, for the gauges and the debug API.+#[derive(Clone, Debug, Default, PartialEq, Eq)]+pub struct GovernorSnapshot {+ pub adaptation: Adaptation,+ pub permits: usize,+ pub ceiling: usize,+ pub free_permits: usize,+ pub starved_messages: usize,+ pub rtt: Option<Duration>,+}++/// An exponentially weighted moving average whose weight follows the time+/// between samples, so a burst of settlements counts as one stretch of time.+#[derive(Clone, Debug)]+struct Ewma {+ time_constant: Duration,+ state: Option<(f64, Instant)>,+}++impl Ewma {+ fn new(time_constant: Duration) -> Self {+ Self {+ time_constant,+ state: None,+ }+ }++ fn observe(&mut self, sample: Duration, now: Instant) {+ let sample = sample.as_secs_f64();+ self.state = Some(match self.state {+ None => (sample, now),+ Some((average, last)) => {+ let elapsed = now.saturating_duration_since(last).as_secs_f64();+ let alpha = 1.0 - (-elapsed / self.time_constant.as_secs_f64()).exp();+ (average + alpha * (sample - average), last.max(now))+ }+ });+ }++ fn value(&self) -> Option<Duration> {+ self.state+ .map(|(average, _)| Duration::from_secs_f64(average))+ }+}++pub struct Governor {+ settings: GovernorSettings,+ max_permits_per_worker: usize,+ rtt: Ewma,+ ceiling: usize,+ floor: usize,+ /// How far the law has shrunk the pool below the ceiling. Kept as a+ /// distance so a worker that joins adds its permits to the pool at once,+ /// and clamped on read so an empty pool does not erase it.+ deficit: usize,+ in_flight: usize,+ starved_messages: usize,+ last_step: Option<Instant>,+}++impl Governor {+ pub fn new(settings: GovernorSettings, max_permits_per_worker: usize) -> Result<Self, String> {+ if settings.min_permits_per_worker == 0 {+ return Err("the governor's min permits per worker must be > 0".to_string());+ }+ if settings.min_permits_per_worker > max_permits_per_worker {+ return Err(+ "the governor's min permits per worker must be at most its max".to_string(),+ );+ }+ if settings.step_interval.is_zero() {+ return Err("the governor's step interval must be > 0".to_string());+ }+ if settings.adaptation == Adaptation::On && settings.latency_budget.is_none() {+ return Err("governor adaptation needs a latency budget".to_string());+ }+ Ok(Self {+ settings,+ max_permits_per_worker,+ rtt: Ewma::new(RTT_TIME_CONSTANT),+ ceiling: 0,+ floor: 0,+ deficit: 0,+ in_flight: 0,+ starved_messages: 0,+ last_step: None,+ })+ }++ pub fn adaptation(&self) -> Adaptation {+ self.settings.adaptation+ }++ pub fn permits(&self) -> usize {+ self.ceiling - self.deficit()+ }++ pub fn ceiling(&self) -> usize {+ self.ceiling+ }++ pub fn free(&self) -> usize {+ self.permits().saturating_sub(self.in_flight)+ }++ fn deficit(&self) -> usize {+ self.deficit.min(self.ceiling - self.floor)+ }++ pub fn rtt(&self) -> Option<Duration> {+ self.rtt.value()+ }++ pub fn snapshot(&self) -> GovernorSnapshot {+ GovernorSnapshot {+ adaptation: self.settings.adaptation,+ permits: self.permits(),+ ceiling: self.ceiling,+ free_permits: self.free(),+ starved_messages: self.starved_messages,+ rtt: self.rtt(),+ }+ }++ /// `in_flight` counts the requests on those workers only: a request on a+ /// worker that left the pool holds none of the pool's permits.+ pub fn set_pool(&mut self, workers: usize, in_flight: usize) {+ self.ceiling = self.max_permits_per_worker * workers;+ self.floor = self.settings.min_permits_per_worker * workers;+ self.in_flight = in_flight;+ }++ /// The ready messages the last pass had beyond its free permits. The pool+ /// grows only while the last pass left some, so a burst that has cleared+ /// grows nothing.+ pub fn set_starved_messages(&mut self, messages: usize) {+ self.starved_messages = messages;+ }++ pub fn observe(&mut self, sample: Duration, now: Instant) -> Option<Decision> {+ self.rtt.observe(sample, now);+ if self+ .last_step+ .is_some_and(|last| now < last + self.settings.step_interval)+ {+ return None;+ }+ let over_budget = match (self.rtt.value(), self.settings.latency_budget) {+ (Some(rtt), Some(budget)) => rtt > budget,+ _ => false,+ };+ let direction = if over_budget {+ Direction::Shrink+ } else if self.starved_messages > 0 {+ Direction::Grow+ } else {+ return None;+ };+ let deficit = self.deficit();+ self.deficit = match direction {+ Direction::Shrink if deficit < self.ceiling - self.floor => deficit + 1,+ Direction::Grow if deficit > 0 => deficit - 1,+ _ => return None,+ };+ self.last_step = Some(now);+ Some(Decision {+ direction,+ applied: self.settings.adaptation == Adaptation::On,+ })+ }+}++#[cfg(test)]+mod tests {+ use rstest::rstest;++ use super::*;++ const STEP: Duration = Duration::from_secs(1);+ const BUDGET: Duration = Duration::from_secs(1);+ const SLOW: Duration = Duration::from_secs(2);+ const FAST: Duration = Duration::from_millis(100);+ const MAX_PER_WORKER: usize = 4;++ fn settings(adaptation: Adaptation) -> GovernorSettings {+ GovernorSettings {+ adaptation,+ latency_budget: Some(BUDGET),+ min_permits_per_worker: 1,+ step_interval: STEP,+ }+ }++ /// Two workers: a ceiling of 8 and a floor of 2.+ fn governor(adaptation: Adaptation) -> Governor {+ let mut governor = Governor::new(settings(adaptation), MAX_PER_WORKER).expect("valid");+ governor.set_pool(2, 0);+ governor+ }++ fn applied(direction: Direction) -> Option<Decision> {+ Some(Decision {+ direction,+ applied: true,+ })+ }++ fn unapplied(direction: Direction) -> Option<Decision> {+ Some(Decision {+ direction,+ applied: false,+ })+ }++ #[test]+ fn the_pool_shrinks_by_one_per_step_while_the_rtt_is_over_budget() {+ let now = Instant::now();+ let mut governor = governor(Adaptation::On);+ assert_eq!(governor.permits(), 8);++ assert_eq!(governor.observe(SLOW, now), applied(Direction::Shrink));+ assert_eq!(governor.permits(), 7);+ assert_eq!(+ governor.observe(SLOW, now + STEP / 2),+ None,+ "no second step inside the interval"+ );+ assert_eq!(governor.permits(), 7);+ assert_eq!(+ governor.observe(SLOW, now + STEP),+ applied(Direction::Shrink)+ );+ assert_eq!(governor.permits(), 6);+ }++ #[test]+ fn the_pool_grows_by_one_per_step_while_the_last_pass_was_starved_and_the_rtt_is_within_budget()+ {+ let now = Instant::now();+ let mut governor = governor(Adaptation::On);+ assert_eq!(governor.observe(SLOW, now), applied(Direction::Shrink));+ assert_eq!(+ governor.observe(SLOW, now + STEP),+ applied(Direction::Shrink)+ );+ assert_eq!(governor.permits(), 6);++ // Long enough for the average to forget the slow samples.+ let recovered = now + RTT_TIME_CONSTANT * 10;+ governor.set_starved_messages(3);+ governor.set_starved_messages(0);+ assert_eq!(+ governor.observe(FAST, recovered),+ None,+ "a starved pass that a later pass cleared grows nothing"+ );++ governor.set_starved_messages(3);+ assert_eq!(+ governor.observe(FAST, recovered + STEP),+ applied(Direction::Grow)+ );+ assert_eq!(+ governor.observe(FAST, recovered + STEP * 2),+ applied(Direction::Grow)+ );+ assert_eq!(governor.permits(), 8);+ }++ #[test]+ fn the_pool_clamps_at_the_floor_and_at_the_ceiling() {+ let now = Instant::now();+ let mut governor = governor(Adaptation::On);+ for step in 0..6 {+ assert_eq!(+ governor.observe(SLOW, now + STEP * step),+ applied(Direction::Shrink)+ );+ }+ assert_eq!(governor.permits(), 2);+ assert_eq!(+ governor.observe(SLOW, now + STEP * 6),+ None,+ "a step at a bound moves nothing, so it reports nothing"+ );+ assert_eq!(governor.permits(), 2);++ let mut governor = self::governor(Adaptation::On);+ governor.set_starved_messages(1);+ assert_eq!(governor.observe(FAST, now), None);+ assert_eq!(governor.permits(), 8);+ }++ #[test]+ fn the_bounds_follow_the_routable_workers_and_keep_the_shrink() {+ let now = Instant::now();+ let mut governor = governor(Adaptation::On);+ assert_eq!(governor.observe(SLOW, now), applied(Direction::Shrink));+ assert_eq!(governor.permits(), 7);++ governor.set_pool(1, 0);+ assert_eq!(governor.ceiling(), 4);+ assert_eq!(governor.permits(), 3, "the ceiling dropped by one worker");++ governor.set_pool(3, 0);+ assert_eq!(governor.ceiling(), 12);+ assert_eq!(+ governor.permits(),+ 11,+ "a joining worker's permits are usable at once"+ );++ for step in 1..=20 {+ governor.observe(SLOW, now + STEP * step);+ }+ assert_eq!(governor.permits(), 3, "at the floor of one per worker");+ governor.set_pool(1, 0);+ assert_eq!(governor.permits(), 1, "the floor dropped with the pool");+ }++ #[test]+ fn an_empty_pool_keeps_the_shrink_until_workers_return() {+ let now = Instant::now();+ let mut governor = governor(Adaptation::On);+ assert_eq!(governor.observe(SLOW, now), applied(Direction::Shrink));+ assert_eq!(+ governor.observe(SLOW, now + STEP),+ applied(Direction::Shrink)+ );+ assert_eq!(governor.permits(), 6);++ governor.set_pool(0, 0);+ assert_eq!((governor.permits(), governor.free()), (0, 0));+ assert_eq!(+ governor.observe(SLOW, now + STEP * 2),+ None,+ "nothing to govern"+ );++ governor.set_pool(2, 0);+ assert_eq!(governor.permits(), 6);+ }++ #[test]+ fn with_adaptation_off_steps_are_unapplied_and_the_shadow_pool_still_moves() {+ let now = Instant::now();+ let mut shadow = governor(Adaptation::Off);+ shadow.set_pool(2, 8);+ assert_eq!(shadow.free(), 0);++ assert_eq!(shadow.observe(SLOW, now), unapplied(Direction::Shrink));+ assert_eq!(shadow.permits(), 7);+ shadow.set_pool(2, 7);+ assert_eq!(+ shadow.free(),+ 0,+ "the shadow pool is at the in-flight count"+ );+ }++ #[test]+ fn the_first_sample_seeds_the_average_and_later_samples_weigh_by_their_spacing() {+ let now = Instant::now();+ let mut ewma = Ewma::new(RTT_TIME_CONSTANT);+ assert_eq!(ewma.value(), None);+ ewma.observe(SLOW, now);+ assert_eq!(ewma.value(), Some(SLOW));++ let mut close = ewma.clone();+ close.observe(Duration::ZERO, now + Duration::from_secs(1));+ let mut apart = ewma;+ apart.observe(Duration::ZERO, now + RTT_TIME_CONSTANT);+ let close = close.value().expect("fed").as_secs_f64();+ let apart = apart.value().expect("fed").as_secs_f64();+ assert!((close - 2.0 * (-0.1f64).exp()).abs() < 0.01, "{close}");+ assert!((apart - 2.0 * (-1.0f64).exp()).abs() < 0.01, "{apart}");+ }++ #[rstest]+ #[case::no_min(GovernorSettings { min_permits_per_worker: 0, ..settings(Adaptation::Off) }, 4)]+ #[case::min_above_max(GovernorSettings { min_permits_per_worker: 5, ..settings(Adaptation::Off) }, 4)]+ #[case::no_step_interval(GovernorSettings { step_interval: Duration::ZERO, ..settings(Adaptation::Off) }, 4)]+ #[case::adaptation_without_a_budget(GovernorSettings { latency_budget: None, ..settings(Adaptation::On) }, 4)]+ fn invalid_settings_are_rejected(+ #[case] settings: GovernorSettings,+ #[case] max_permits_per_worker: usize,+ ) {+ assert!(Governor::new(settings, max_permits_per_worker).is_err());+ }+}diff --git a/rust/ingestion-consumer/src/batcher/mod.rs b/rust/ingestion-consumer/src/batcher/mod.rsindex 9fa784cc8f1a..fb552d784007 100644--- a/rust/ingestion-consumer/src/batcher/mod.rs+++ b/rust/ingestion-consumer/src/batcher/mod.rs@@ -14,6 +14,7 @@//! the failure decision stays in the consumer loop.mod driver;+pub mod governor;pub mod in_flight;pub mod key_queues;pub mod packer;