Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

24.5 KB
Raw Blame
//! App layer: validated orchestrator/producer tunables parsed once from [`Config`]. Depends on
//! `config`, `domain`, `store`, and its `app` siblings; this `TryFrom<&Config>` is what keeps the
//! dependency arrow pointing down (`config` no longer names an `app` type).

use std::num::{NonZeroU16, NonZeroU32, NonZeroU64, NonZeroUsize};
use std::time::Duration;

use crate::config::Config;
use crate::domain::{BackoffPolicyError, PersonEmissionPolicy, PlanCaps, RetryBackoffPolicy};
use crate::store::runs::RunKind;
use crate::store::{LeaseDuration, LeaseDurationError, MaxAttempts, MaxAttemptsError};

use super::breaker::{BreakerPolicy, BreakerPolicyError};
use super::deliver::QUEUE_FULL_BACKOFF_CAP;
use super::orchestrator::ORCHESTRATOR_LIVENESS_DEADLINE;

/// The person seed path's tunables. `OrchestratorSettings.person` is `None` when
/// `SEEDER_PERSON_SEEDS_ENABLED` is off — the type encodes "person path off".
#[derive(Debug, Clone, Copy)]
pub struct PersonSettings {
    pub seeds_per_sec: NonZeroU32,
    pub persons_per_chunk: NonZeroU64,
    /// The person path's own slot budget: a person chunk never occupies a behavioral slot, so a
    /// long person scan cannot stall live behavioral seeding.
    pub max_concurrent_chunks: NonZeroUsize,
    /// Which scanned persons a chunk emits. `SEEDER_PERSON_EMIT_NONMATCHERS` names the choice.
    pub emission: PersonEmissionPolicy,
    /// Whether completion discovery may surface person runs. Separate from the seed gate so the
    /// reconcile half can be staged after the processor fleet decodes `reconcile_person` tiles —
    /// seeding a person run is inert, dispatching its tiles to an old processor is not.
    pub reconcile_dispatch: bool,
}

impl PersonSettings {
    /// The pacer is shared across concurrent person chunks, so one chunk's floor wall-clock is
    /// `persons_per_chunk / (seeds_per_sec / concurrency)` — and the ClickHouse cursor stays open
    /// across the paced produce, counting client read time against `max_execution_time`. Refuse a
    /// combination that would eat more than half that budget at startup rather than at hour four.
    fn validate_scan_budget(
        &self,
        ch_max_execution_time_secs: u64,
    ) -> Result<(), OrchestratorSettingsError> {
        let projected_secs = self
            .persons_per_chunk
            .get()
            .saturating_mul(self.max_concurrent_chunks.get() as u64)
            / u64::from(self.seeds_per_sec.get());
        let budget_secs = ch_max_execution_time_secs / 2;
        if projected_secs > budget_secs {
            return Err(OrchestratorSettingsError::PersonChunkExceedsScanBudget {
                projected_secs,
                budget_secs,
            });
        }
        Ok(())
    }
}

#[derive(Debug, Clone, Copy)]
pub struct OrchestratorSettings {
    pub(super) run_poll_interval: Duration,
    pub(super) max_concurrent_chunks: NonZeroUsize,
    pub(super) chunk_lease: LeaseDuration,
    pub(super) max_chunk_attempts: MaxAttempts,
    pub(super) retry_backoff: RetryBackoffPolicy,
    pub(super) breaker: BreakerPolicy,
    pub(super) plan_caps: PlanCaps,
    pub(super) producer: ProducerSettings,
    pub(super) person: Option<PersonSettings>,
    pub(super) newest_day_index_max_entries: NonZeroUsize,
}

impl OrchestratorSettings {
    #[allow(clippy::too_many_arguments)]
    pub fn new(
        run_poll_interval: Duration,
        max_concurrent_chunks: usize,
        chunk_lease: Duration,
        max_chunk_attempts: u32,
        retry_backoff: RetryBackoffPolicy,
        breaker: BreakerPolicy,
        max_lookback_days: u32,
        bands_per_day: u16,
        live_tracking_lag: Duration,
        trailing_day_grace: Duration,
        producer: ProducerSettings,
        person: Option<PersonSettings>,
        newest_day_index_max_entries: usize,
    ) -> Result<Self, OrchestratorSettingsError> {
        if run_poll_interval.is_zero() {
            return Err(OrchestratorSettingsError::ZeroPollInterval);
        }
        if run_poll_interval >= ORCHESTRATOR_LIVENESS_DEADLINE {
            return Err(OrchestratorSettingsError::PollIntervalExceedsLivenessDeadline);
        }
        let max_concurrent_chunks = NonZeroUsize::new(max_concurrent_chunks)
            .ok_or(OrchestratorSettingsError::ZeroConcurrency)?;
        let chunk_lease = LeaseDuration::new(chunk_lease).map_err(|error| match error {
            LeaseDurationError::TooShort => OrchestratorSettingsError::LeaseTooShort,
            LeaseDurationError::TooLong => OrchestratorSettingsError::LeaseTooLong,
        })?;
        let max_chunk_attempts =
            MaxAttempts::new(max_chunk_attempts).map_err(|error| match error {
                MaxAttemptsError::Zero => OrchestratorSettingsError::ZeroMaxAttempts,
                MaxAttemptsError::OutOfRange(_) => OrchestratorSettingsError::MaxAttemptsOutOfRange,
            })?;
        // The `band` column is a smallint, so the count must fit i16 for band indices to encode.
        let bands_per_day = NonZeroU16::new(bands_per_day)
            .filter(|bands| i16::try_from(bands.get()).is_ok())
            .ok_or(OrchestratorSettingsError::BandsPerDayOutOfRange)?;
        let newest_day_index_max_entries = NonZeroUsize::new(newest_day_index_max_entries)
            .ok_or(OrchestratorSettingsError::ZeroNewestDayIndexEntries)?;
        Ok(Self {
            run_poll_interval,
            max_concurrent_chunks,
            chunk_lease,
            max_chunk_attempts,
            retry_backoff,
            breaker,
            plan_caps: PlanCaps {
                max_lookback_days,
                bands_per_day,
                live_tracking_lag,
                trailing_day_grace,
            },
            producer,
            person,
            newest_day_index_max_entries,
        })
    }

    pub fn person(&self) -> Option<&PersonSettings> {
        self.person.as_ref()
    }

    /// The backfill kinds run discovery binds: the person seed path is on or it is not.
    pub fn seed_kinds(&self) -> &'static [RunKind] {
        if self.person.is_some() {
            &[RunKind::Behavioral, RunKind::PersonProperty]
        } else {
            &[RunKind::Behavioral]
        }
    }

    /// The backfill kinds completion discovery binds. Narrower than [`Self::seed_kinds`] on
    /// purpose: seeding a person run is inert, dispatching its reconcile tiles to a processor that
    /// cannot decode them is not, so the reconcile gate stages after the seed gate.
    pub fn completion_kinds(&self) -> &'static [RunKind] {
        if self.person.is_some_and(|person| person.reconcile_dispatch) {
            &[RunKind::Behavioral, RunKind::PersonProperty]
        } else {
            &[RunKind::Behavioral]
        }
    }
}

impl TryFrom<&Config> for OrchestratorSettings {
    type Error = SettingsError;

    fn try_from(config: &Config) -> Result<Self, Self::Error> {
        let producer = ProducerSettings::new(
            config.seeder_max_inflight_tiles,
            Duration::from_millis(config.seeder_queue_full_backoff_ms),
        )?;
        let person = config
            .seeder_person_seeds_enabled
            .then(|| {
                let person = PersonSettings {
                    seeds_per_sec: NonZeroU32::new(config.seeder_person_seeds_per_sec)
                        .ok_or(OrchestratorSettingsError::ZeroPersonSeedRate)?,
                    persons_per_chunk: NonZeroU64::new(config.seeder_persons_per_chunk)
                        .ok_or(OrchestratorSettingsError::ZeroPersonsPerChunk)?,
                    max_concurrent_chunks: NonZeroUsize::new(
                        config.seeder_person_max_concurrent_chunks,
                    )
                    .ok_or(OrchestratorSettingsError::ZeroPersonConcurrency)?,
                    emission: PersonEmissionPolicy::from_emit_nonmatchers(
                        config.seeder_person_emit_nonmatchers,
                    ),
                    reconcile_dispatch: config.seeder_person_reconcile_dispatch_enabled,
                };
                person.validate_scan_budget(config.seeder_ch_max_execution_time_secs)?;
                Ok::<_, OrchestratorSettingsError>(person)
            })
            .transpose()?;
        let retry_backoff = RetryBackoffPolicy::new(
            Duration::from_secs(config.seeder_retry_backoff_base_secs),
            Duration::from_secs(config.seeder_retry_backoff_cap_secs),
        )
        .map_err(OrchestratorSettingsError::RetryBackoff)?;
        let breaker = BreakerPolicy::new(
            config.seeder_ch_breaker_threshold,
            Duration::from_secs(config.seeder_ch_breaker_cooldown_base_secs),
            Duration::from_secs(config.seeder_ch_breaker_cooldown_cap_secs),
            config.seeder_ch_breaker_max_trips,
        )
        .map_err(OrchestratorSettingsError::Breaker)?;
        Ok(Self::new(
            Duration::from_secs(config.seeder_run_poll_secs),
            config.seeder_max_concurrent_chunks,
            Duration::from_secs(config.seeder_chunk_lease_secs),
            config.seeder_max_chunk_attempts,
            retry_backoff,
            breaker,
            config.seeder_max_lookback_days,
            config.seeder_bands_per_day,
            Duration::from_secs(config.seeder_live_tracking_lag_secs),
            Duration::from_secs(config.seeder_trailing_day_grace_secs),
            producer,
            person,
            config.seeder_newest_day_index_max_entries,
        )?)
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
pub enum OrchestratorSettingsError {
    #[error("run poll interval must be greater than zero")]
    ZeroPollInterval,
    #[error("run poll interval must be shorter than the liveness deadline")]
    PollIntervalExceedsLivenessDeadline,
    #[error("maximum concurrent chunks must be greater than zero")]
    ZeroConcurrency,
    #[error("chunk lease must be at least three seconds")]
    LeaseTooShort,
    #[error("chunk lease exceeds PostgreSQL interval range")]
    LeaseTooLong,
    #[error("maximum chunk attempts must be greater than zero")]
    ZeroMaxAttempts,
    #[error("maximum chunk attempts exceeds PostgreSQL integer range")]
    MaxAttemptsOutOfRange,
    #[error("bands per day must be between 1 and 32767")]
    BandsPerDayOutOfRange,
    #[error(transparent)]
    RetryBackoff(#[from] BackoffPolicyError),
    #[error(transparent)]
    Breaker(#[from] BreakerPolicyError),
    #[error("person seeds per second must be greater than zero")]
    ZeroPersonSeedRate,
    #[error("persons per chunk must be greater than zero")]
    ZeroPersonsPerChunk,
    #[error("person max concurrent chunks must be greater than zero")]
    ZeroPersonConcurrency,
    #[error("newest-day index max entries must be greater than zero")]
    ZeroNewestDayIndexEntries,
    #[error(
        "one person chunk needs ~{projected_secs}s at the configured rate and concurrency, over \
         the {budget_secs}s scan budget (half of SEEDER_CH_MAX_EXECUTION_TIME_SECS); shrink \
         SEEDER_PERSONS_PER_CHUNK or raise SEEDER_PERSON_SEEDS_PER_SEC"
    )]
    PersonChunkExceedsScanBudget {
        projected_secs: u64,
        budget_secs: u64,
    },
}

/// The produce sequencing's tunables: the in-flight delivery bound and the queue-full backoff (capped
/// at [`QUEUE_FULL_BACKOFF_CAP`]). Consumed by `app::deliver`.
#[derive(Debug, Clone, Copy)]
pub struct ProducerSettings {
    pub(super) max_inflight: NonZeroUsize,
    pub(super) queue_full_backoff: Duration,
}

impl ProducerSettings {
    pub fn new(
        max_inflight: usize,
        queue_full_backoff: Duration,
    ) -> Result<Self, ProducerSettingsError> {
        let max_inflight =
            NonZeroUsize::new(max_inflight).ok_or(ProducerSettingsError::ZeroMaxInflight)?;
        if queue_full_backoff.is_zero() {
            return Err(ProducerSettingsError::ZeroQueueFullBackoff);
        }
        Ok(Self {
            max_inflight,
            queue_full_backoff: queue_full_backoff.min(QUEUE_FULL_BACKOFF_CAP),
        })
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
pub enum ProducerSettingsError {
    #[error("maximum in-flight tiles must be greater than zero")]
    ZeroMaxInflight,
    #[error("queue-full backoff must be greater than zero")]
    ZeroQueueFullBackoff,
}

/// Folds the producer- and orchestrator-settings validations behind one `TryFrom<&Config>` error, so
/// the seeder binary sees a single failure type when parsing its environment.
#[derive(Debug, thiserror::Error)]
pub enum SettingsError {
    #[error(transparent)]
    Producer(#[from] ProducerSettingsError),
    #[error(transparent)]
    Orchestrator(#[from] OrchestratorSettingsError),
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use envconfig::Envconfig;

    use super::*;

    fn producer_settings() -> ProducerSettings {
        ProducerSettings::new(1, Duration::from_millis(1)).unwrap()
    }

    fn backoff() -> RetryBackoffPolicy {
        RetryBackoffPolicy::new(Duration::from_secs(30), Duration::from_secs(1800)).unwrap()
    }

    fn breaker() -> BreakerPolicy {
        BreakerPolicy::new(3, Duration::from_secs(300), Duration::from_secs(1800), 4).unwrap()
    }

    #[test]
    fn producer_settings_reject_unbounded_bounds() {
        assert_eq!(
            ProducerSettings::new(0, Duration::from_millis(1)).unwrap_err(),
            ProducerSettingsError::ZeroMaxInflight
        );
        assert_eq!(
            ProducerSettings::new(1, Duration::ZERO).unwrap_err(),
            ProducerSettingsError::ZeroQueueFullBackoff
        );
        assert_eq!(
            ProducerSettings::new(1, Duration::from_secs(60))
                .unwrap()
                .queue_full_backoff,
            QUEUE_FULL_BACKOFF_CAP
        );
    }

    #[test]
    fn settings_reject_values_that_disable_progress_or_lease_heartbeats() {
        let cases = [
            (
                OrchestratorSettings::new(
                    ORCHESTRATOR_LIVENESS_DEADLINE,
                    1,
                    Duration::from_secs(3),
                    1,
                    backoff(),
                    breaker(),
                    400,
                    1,
                    Duration::ZERO,
                    Duration::ZERO,
                    producer_settings(),
                    None,
                    1,
                ),
                OrchestratorSettingsError::PollIntervalExceedsLivenessDeadline,
            ),
            (
                OrchestratorSettings::new(
                    Duration::ZERO,
                    1,
                    Duration::from_secs(3),
                    1,
                    backoff(),
                    breaker(),
                    400,
                    1,
                    Duration::ZERO,
                    Duration::ZERO,
                    producer_settings(),
                    None,
                    1,
                ),
                OrchestratorSettingsError::ZeroPollInterval,
            ),
            (
                OrchestratorSettings::new(
                    Duration::from_secs(1),
                    0,
                    Duration::from_secs(3),
                    1,
                    backoff(),
                    breaker(),
                    400,
                    1,
                    Duration::ZERO,
                    Duration::ZERO,
                    producer_settings(),
                    None,
                    1,
                ),
                OrchestratorSettingsError::ZeroConcurrency,
            ),
            (
                OrchestratorSettings::new(
                    Duration::from_secs(1),
                    1,
                    Duration::from_secs(2),
                    1,
                    backoff(),
                    breaker(),
                    400,
                    1,
                    Duration::ZERO,
                    Duration::ZERO,
                    producer_settings(),
                    None,
                    1,
                ),
                OrchestratorSettingsError::LeaseTooShort,
            ),
            (
                OrchestratorSettings::new(
                    Duration::from_secs(1),
                    1,
                    Duration::from_secs(3),
                    0,
                    backoff(),
                    breaker(),
                    400,
                    1,
                    Duration::ZERO,
                    Duration::ZERO,
                    producer_settings(),
                    None,
                    1,
                ),
                OrchestratorSettingsError::ZeroMaxAttempts,
            ),
            (
                OrchestratorSettings::new(
                    Duration::from_secs(1),
                    1,
                    Duration::from_secs(3),
                    1,
                    backoff(),
                    breaker(),
                    400,
                    0,
                    Duration::ZERO,
                    Duration::ZERO,
                    producer_settings(),
                    None,
                    1,
                ),
                OrchestratorSettingsError::BandsPerDayOutOfRange,
            ),
            (
                OrchestratorSettings::new(
                    Duration::from_secs(1),
                    1,
                    Duration::from_secs(3),
                    1,
                    backoff(),
                    breaker(),
                    400,
                    u16::MAX,
                    Duration::ZERO,
                    Duration::ZERO,
                    producer_settings(),
                    None,
                    1,
                ),
                OrchestratorSettingsError::BandsPerDayOutOfRange,
            ),
        ];
        for (result, expected) in cases {
            assert_eq!(result.unwrap_err(), expected);
        }
    }

    /// Dark-by-default: with the gates off discovery binds behavioral only; enabling them validates
    /// the person rates the way `ZeroTileRate` guards the behavioral pacer. The two gates stage
    /// separately — seeding a person run is inert, dispatching its tiles to a processor that cannot
    /// decode the person kind is not, and that run cannot be recovered by flipping the gate back.
    #[test]
    fn person_settings_are_gated_and_validated() {
        let config = Config::init_from_hashmap(&HashMap::new()).unwrap();
        let dark = OrchestratorSettings::try_from(&config).unwrap();
        assert!(dark.person().is_none());
        assert_eq!(dark.seed_kinds(), &[RunKind::Behavioral]);
        assert_eq!(dark.completion_kinds(), &[RunKind::Behavioral]);

        // The reconcile gate alone arms nothing: there is no person seed path to reconcile.
        let mut reconcile_only = config.clone();
        reconcile_only.seeder_person_reconcile_dispatch_enabled = true;
        let reconcile_only = OrchestratorSettings::try_from(&reconcile_only).unwrap();
        assert!(reconcile_only.person().is_none());
        assert_eq!(reconcile_only.seed_kinds(), &[RunKind::Behavioral]);
        assert_eq!(reconcile_only.completion_kinds(), &[RunKind::Behavioral]);

        let mut enabled = config.clone();
        enabled.seeder_person_seeds_enabled = true;
        // Seeding without dispatch is the staging step: person runs are discovered and seed, then
        // park in `seeding` because completion discovery still never surfaces them.
        let seeds_only = OrchestratorSettings::try_from(&enabled).unwrap();
        assert!(seeds_only.person().is_some());
        assert_eq!(
            seeds_only.seed_kinds(),
            &[RunKind::Behavioral, RunKind::PersonProperty]
        );
        assert_eq!(seeds_only.completion_kinds(), &[RunKind::Behavioral]);

        let mut both = enabled.clone();
        both.seeder_person_reconcile_dispatch_enabled = true;
        let lit = OrchestratorSettings::try_from(&both).unwrap();
        assert!(lit.person().is_some());
        assert_eq!(
            lit.seed_kinds(),
            &[RunKind::Behavioral, RunKind::PersonProperty]
        );
        assert_eq!(
            lit.completion_kinds(),
            &[RunKind::Behavioral, RunKind::PersonProperty]
        );

        enabled.seeder_person_seeds_per_sec = 0;
        assert!(matches!(
            OrchestratorSettings::try_from(&enabled),
            Err(SettingsError::Orchestrator(
                OrchestratorSettingsError::ZeroPersonSeedRate
            ))
        ));
        enabled.seeder_person_seeds_per_sec = 2000;
        enabled.seeder_persons_per_chunk = 0;
        assert!(matches!(
            OrchestratorSettings::try_from(&enabled),
            Err(SettingsError::Orchestrator(
                OrchestratorSettingsError::ZeroPersonsPerChunk
            ))
        ));
        enabled.seeder_persons_per_chunk = 1_000_000;
        enabled.seeder_person_max_concurrent_chunks = 0;
        assert!(matches!(
            OrchestratorSettings::try_from(&enabled),
            Err(SettingsError::Orchestrator(
                OrchestratorSettingsError::ZeroPersonConcurrency
            ))
        ));
    }

    /// A chunk whose floor wall-clock (shared pacer, concurrency multiplier) would eat more than
    /// half the ClickHouse execution-time budget must refuse at startup, not at hour four.
    #[test]
    fn person_chunk_sizing_must_fit_the_clickhouse_scan_budget() {
        let mut config = Config::init_from_hashmap(&HashMap::new()).unwrap();
        config.seeder_person_seeds_enabled = true;
        assert!(OrchestratorSettings::try_from(&config).is_ok());

        // 5M persons at 2000/s across 6 concurrent chunks ⇒ 15000s > 14400/2.
        config.seeder_persons_per_chunk = 5_000_000;
        config.seeder_person_max_concurrent_chunks = 6;
        assert!(matches!(
            OrchestratorSettings::try_from(&config),
            Err(SettingsError::Orchestrator(
                OrchestratorSettingsError::PersonChunkExceedsScanBudget {
                    projected_secs: 15_000,
                    budget_secs: 7_200,
                }
            ))
        ));
    }

    #[test]
    fn breaker_is_parsed_from_config_and_rejected_when_it_cannot_trip() {
        let config = Config::init_from_hashmap(&HashMap::new()).unwrap();
        let settings = OrchestratorSettings::try_from(&config).unwrap();
        assert_eq!(
            settings.breaker,
            BreakerPolicy::new(3, Duration::from_secs(300), Duration::from_secs(1800), 4).unwrap()
        );

        let mut zero_threshold = config.clone();
        zero_threshold.seeder_ch_breaker_threshold = 0;
        assert!(matches!(
            OrchestratorSettings::try_from(&zero_threshold),
            Err(SettingsError::Orchestrator(
                OrchestratorSettingsError::Breaker(BreakerPolicyError::ZeroThreshold)
            ))
        ));
    }

    /// A backoff pair that cannot space retries must fail at startup. Accepting it would leave the
    /// claim gate stamping a zero or shrinking wait, which is the reclaim storm the policy exists
    /// to stop, and nothing downstream would report it.
    #[test]
    fn retry_backoff_is_parsed_from_config_and_rejected_when_it_cannot_space_retries() {
        let config = Config::init_from_hashmap(&HashMap::new()).unwrap();
        let settings = OrchestratorSettings::try_from(&config).unwrap();
        assert_eq!(settings.retry_backoff.base(), Duration::from_secs(30));
        assert_eq!(settings.retry_backoff.cap(), Duration::from_secs(1800));
        assert_eq!(
            settings.plan_caps.live_tracking_lag,
            Duration::from_secs(420)
        );
        assert_eq!(
            settings.plan_caps.trailing_day_grace,
            Duration::from_secs(1800)
        );

        let mut zero_base = config.clone();
        zero_base.seeder_retry_backoff_base_secs = 0;
        assert!(matches!(
            OrchestratorSettings::try_from(&zero_base),
            Err(SettingsError::Orchestrator(
                OrchestratorSettingsError::RetryBackoff(BackoffPolicyError::ZeroBase)
            ))
        ));

        let mut cap_below_base = config.clone();
        cap_below_base.seeder_retry_backoff_cap_secs = 29;
        assert!(matches!(
            OrchestratorSettings::try_from(&cap_below_base),
            Err(SettingsError::Orchestrator(
                OrchestratorSettingsError::RetryBackoff(BackoffPolicyError::CapBelowBase)
            ))
        ));
    }

    #[test]
    fn try_from_config_rejects_disabled_concurrency() {
        let mut config = Config::init_from_hashmap(&HashMap::new()).unwrap();
        config.seeder_max_concurrent_chunks = 0;
        assert!(matches!(
            OrchestratorSettings::try_from(&config),
            Err(SettingsError::Orchestrator(
                OrchestratorSettingsError::ZeroConcurrency
            ))
        ));
    }
}