Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

16.0 KB
Raw Blame
//! Which teams the realtime cohort pipeline processes.
//!
//! A team processes when `REALTIME_COHORT_TEAM_ALLOWLIST` names it, or when its row in Django's
//! `cohort_realtime_team_access` table is at `processing` or `serving`. [`TeamAccessPoller`] reads
//! the whole table on a short poll and swaps the rows into [`TeamAccess`], so a grant or a revoke
//! reaches every reader within one poll, and the per-event check never waits on a lock.

use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};

use arc_swap::ArcSwapOption;
use common_types::cohort::TeamAllowlist;
use lifecycle::Handle;
use metrics::gauge;
use rand::Rng;
use sqlx::PgPool;
use thiserror::Error;
use tracing::{debug, warn};

/// A transaction-mode PgBouncer does not keep the session `statement_timeout` that
/// `common-database` sets after connect. So each read sets its own timeout inside its transaction,
/// and the server cancels a blocked read instead of holding a pool connection.
const QUERY_TIMEOUT_SQL: &str = "SET LOCAL statement_timeout = '5s'";
const ACCESS_ROWS_SQL: &str = "SELECT team_id, level FROM cohort_realtime_team_access";

/// Bounds the whole poll on the client. The statement timeout stops only the server, so a socket
/// that dies after the pool hands it out would otherwise hang the poll and freeze every grant and
/// revoke. Longer than the statement timeout, so the server cancels a slow read first.
const POLL_TIMEOUT: Duration = Duration::from_secs(10);

const POLL_INTERVAL: Duration = Duration::from_secs(15);
const POLL_JITTER: Duration = Duration::from_secs(3);

/// Zero until the first poll succeeds, then set only when a poll succeeds. So a staleness alert
/// covers a reader that never loaded, and failed polls, including those that keep a regressed row.
/// The charts allowlist of metrics that alerts can read matches the `posthog_cohort_` prefix.
const LAST_SUCCESS: &str = "posthog_cohort_team_access_last_success_timestamp_seconds";

#[derive(Debug, Error)]
pub enum PollError {
    #[error("querying cohort_realtime_team_access failed: {0}")]
    Query(#[from] sqlx::Error),
    #[error("reading cohort_realtime_team_access timed out")]
    TimedOut(#[from] tokio::time::error::Elapsed),
    #[error("team {team_id} has the unknown level {level:?}")]
    UnknownLevel { team_id: i32, level: String },
    /// Such a read is wrong: an empty table, another database, or a replica that lags behind the
    /// previous read. See [`Level::may_become`]. These teams keep their previous rows, and the
    /// read's other rows still take effect, so a revoke reaches the reader even then.
    #[error("the read lacks the rows of teams {0:?}, or grants them again after `off`, so they keep their previous rows")]
    RegressedRows(Vec<i32>),
}

/// The levels of `RealtimeCohortLevel` in `products/cohorts/backend/models/team_access.py`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Level {
    Off,
    Processing,
    Serving,
}

impl Level {
    fn parse(level: &str) -> Option<Self> {
        match level {
            "off" => Some(Self::Off),
            "processing" => Some(Self::Processing),
            "serving" => Some(Self::Serving),
            _ => None,
        }
    }

    fn processes(self) -> bool {
        match self {
            Self::Off => false,
            Self::Processing | Self::Serving => true,
        }
    }

    /// Whether a later read may show this row at `next`. Django never deletes a row, and it refuses
    /// to grant an `off` row again, so only a wrong read shows either change.
    fn may_become(self, next: Option<Self>) -> bool {
        match (self, next) {
            (_, None) => false,
            (Self::Off, Some(Self::Off)) => true,
            (Self::Off, Some(Self::Processing | Self::Serving)) => false,
            (Self::Processing | Self::Serving, Some(_)) => true,
        }
    }
}

/// Every row of one read, `off` rows included, because the poller checks the next read against them.
#[derive(Debug)]
struct AccessRows(HashMap<i32, Level>);

impl AccessRows {
    fn parse(rows: Vec<(i32, String)>) -> Result<Self, PollError> {
        rows.into_iter()
            .map(|(team_id, level)| match Level::parse(&level) {
                Some(parsed) => Ok((team_id, parsed)),
                None => Err(PollError::UnknownLevel { team_id, level }),
            })
            .collect::<Result<_, _>>()
            .map(Self)
    }

    fn granted(&self) -> impl Iterator<Item = i32> + '_ {
        self.0
            .iter()
            .filter(|(_, level)| level.processes())
            .map(|(&team_id, _)| team_id)
    }

    /// Puts back the `previous` row of each team whose row here is a change that
    /// [`Level::may_become`] rules out, and returns those teams.
    fn restore_regressed(&mut self, previous: &Self) -> Vec<i32> {
        let mut regressed = Vec::new();
        for (&team_id, &level) in &previous.0 {
            if !level.may_become(self.0.get(&team_id).copied()) {
                self.0.insert(team_id, level);
                regressed.push(team_id);
            }
        }
        regressed.sort_unstable();
        regressed
    }
}

/// The teams the pipeline processes after an accepted read: the env list united with the teams
/// whose row is at `processing` or `serving`. Only an accepted read builds one, so a reader that
/// holds one never runs on the env list alone.
#[derive(Debug, PartialEq, Eq)]
pub struct ProcessingTeams(TeamAllowlist);

impl ProcessingTeams {
    /// An `off` row never removes a team that the env list names.
    fn new(env: &TeamAllowlist, rows: &AccessRows) -> Self {
        Self(match env {
            TeamAllowlist::All => TeamAllowlist::All,
            TeamAllowlist::Only(listed) => {
                TeamAllowlist::Only(listed.iter().copied().chain(rows.granted()).collect())
            }
        })
    }

    pub fn scope(&self) -> &TeamAllowlist {
        &self.0
    }
}

/// One accepted read. The poller checks the next read against `rows`, and readers ask `processing`.
#[derive(Debug)]
struct Snapshot {
    rows: AccessRows,
    processing: Arc<ProcessingTeams>,
}

impl Snapshot {
    fn new(env: &TeamAllowlist, rows: AccessRows) -> Self {
        let processing = Arc::new(ProcessingTeams::new(env, &rows));
        Self { rows, processing }
    }
}

/// Answers whether a team processes, from the read the poller accepted last.
#[derive(Debug)]
pub struct TeamAccess {
    env: TeamAllowlist,
    /// `None` until the first accepted read. Until then no team processes, not even one the env list
    /// names: a reader that ran on the env list alone would treat every granted team as revoked.
    snapshot: ArcSwapOption<Snapshot>,
}

impl TeamAccess {
    pub fn new(env: TeamAllowlist) -> (Arc<Self>, TeamAccessPoller) {
        gauge!(LAST_SUCCESS).set(0.0);
        let access = Arc::new(Self {
            env,
            snapshot: ArcSwapOption::empty(),
        });
        let poller = TeamAccessPoller {
            access: access.clone(),
        };
        (access, poller)
    }

    /// A loaded reader whose one read granted `granted` at `processing`.
    #[cfg(feature = "test-util")]
    pub fn with_granted(env: TeamAllowlist, granted: impl IntoIterator<Item = i32>) -> Arc<Self> {
        let (access, mut poller) = Self::new(env);
        poller
            .accept_granted(granted)
            .expect("a first read has no previous rows to regress");
        access
    }

    pub fn processes(&self, team_id: i32) -> bool {
        self.snapshot
            .load()
            .as_ref()
            .is_some_and(|snapshot| snapshot.processing.scope().includes(team_id))
    }

    /// `None` until the first accepted read.
    pub fn processing_teams(&self) -> Option<Arc<ProcessingTeams>> {
        self.snapshot
            .load()
            .as_ref()
            .map(|snapshot| snapshot.processing.clone())
    }

    pub fn is_loaded(&self) -> bool {
        self.snapshot.load().is_some()
    }
}

/// The only writer of its [`TeamAccess`]. It checks each read against the rows it stored last,
/// which is sound only while one poller swaps the rows, so it is not `Clone`.
#[derive(Debug)]
pub struct TeamAccessPoller {
    access: Arc<TeamAccess>,
}

impl TeamAccessPoller {
    /// Reads the table and swaps the rows in. Returns how many rows grant processing.
    ///
    /// # Errors
    ///
    /// Fails when the query fails or times out, or when a row has a level this crate does not know.
    /// The previous rows then stay in place. Also fails when the read lacks a row that the reader
    /// holds, or grants one of its `off` rows again. Only those teams keep their previous rows.
    /// So a row deleted or granted again by hand keeps its old state, and fails every later poll,
    /// until the service restarts.
    pub async fn poll(&mut self, pool: &PgPool) -> Result<usize, PollError> {
        let read = async {
            let mut transaction = pool.begin().await?;
            sqlx::query(QUERY_TIMEOUT_SQL)
                .execute(&mut *transaction)
                .await?;
            let fetched = sqlx::query_as::<_, (i32, String)>(ACCESS_ROWS_SQL)
                .fetch_all(&mut *transaction)
                .await?;
            transaction.commit().await?;
            Ok::<_, sqlx::Error>(fetched)
        };
        let fetched = tokio::time::timeout(POLL_TIMEOUT, read).await??;
        self.accept(AccessRows::parse(fetched)?)
    }

    fn accept(&mut self, mut read: AccessRows) -> Result<usize, PollError> {
        let regressed = match self.access.snapshot.load().as_deref() {
            Some(previous) => read.restore_regressed(&previous.rows),
            None => Vec::new(),
        };
        let granted = read.granted().count();
        let snapshot = Snapshot::new(&self.access.env, read);
        self.access.snapshot.store(Some(Arc::new(snapshot)));
        if !regressed.is_empty() {
            return Err(PollError::RegressedRows(regressed));
        }
        gauge!(LAST_SUCCESS).set(now_unix_seconds());
        Ok(granted)
    }

    /// Polls every `POLL_INTERVAL ± POLL_JITTER` until shutdown. After a failed poll it waits for
    /// the next tick. Shutdown also cancels a poll in flight, because a poll can last longer than
    /// the graceful shutdown window of the service. A cancelled poll changes no rows.
    ///
    /// Register `handle` without a liveness deadline: a database outage must not stop a service that
    /// can keep working on the rows it has.
    pub async fn run(mut self, pool: PgPool, handle: Handle) {
        let _guard = handle.process_scope();

        loop {
            let tick = async {
                tokio::time::sleep(next_poll_delay()).await;
                self.poll(&pool).await
            };
            tokio::select! {
                _ = handle.shutdown_recv() => break,
                polled = tick => match polled {
                    Ok(granted) => debug!(granted_teams = granted, "team access polled"),
                    Err(err) => warn!(error = %err, "team access poll failed"),
                },
            }
        }
    }
}

#[cfg(feature = "test-util")]
impl TeamAccessPoller {
    /// Accepts a read whose rows grant `granted` at `processing`, as a poll would.
    pub fn accept_granted(
        &mut self,
        granted: impl IntoIterator<Item = i32>,
    ) -> Result<usize, PollError> {
        let rows = granted
            .into_iter()
            .map(|team_id| (team_id, Level::Processing))
            .collect();
        self.accept(AccessRows(rows))
    }
}

/// `POLL_INTERVAL ± POLL_JITTER`. Every reader polls at this cadence, and the jitter spreads the
/// polls of many pods over time.
pub fn next_poll_delay() -> Duration {
    let max_jitter_ms = 2 * POLL_JITTER.as_millis() as u64;
    let jitter = Duration::from_millis(rand::thread_rng().gen_range(0..=max_jitter_ms));
    POLL_INTERVAL - POLL_JITTER + jitter
}

/// Seconds since the Unix epoch, saturating at 0 on a clock set before it.
fn now_unix_seconds() -> f64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|since| since.as_secs_f64())
        .unwrap_or(0.0)
}

#[cfg(test)]
mod tests {
    use std::collections::HashSet;

    use super::*;
    use metrics_util::debugging::DebuggingRecorder;

    fn rows(rows: &[(i32, &str)]) -> AccessRows {
        AccessRows::parse(
            rows.iter()
                .map(|&(team_id, level)| (team_id, level.to_string()))
                .collect(),
        )
        .unwrap()
    }

    fn env_only(teams: &[i32]) -> TeamAllowlist {
        TeamAllowlist::Only(teams.iter().copied().collect())
    }

    fn sets_last_success<T>(f: impl FnOnce() -> T) -> (T, bool) {
        let recorder = DebuggingRecorder::new();
        let snapshotter = recorder.snapshotter();
        let out = metrics::with_local_recorder(&recorder, f);
        let set = snapshotter
            .snapshot()
            .into_vec()
            .iter()
            .any(|(key, ..)| key.key().name() == LAST_SUCCESS);
        (out, set)
    }

    #[test]
    fn no_team_processes_before_the_first_accepted_read() {
        let (access, _poller) = TeamAccess::new(TeamAllowlist::All);

        assert!(!access.is_loaded());
        assert!(!access.processes(2));
        assert!(access.processing_teams().is_none());
    }

    #[test]
    fn processing_is_the_env_list_united_with_the_granted_rows() {
        let read = [(3, "off"), (5, "processing"), (6, "serving"), (7, "off")];
        for (env, expected_scope) in [
            (
                env_only(&[2, 3]),
                TeamAllowlist::Only(HashSet::from([2, 3, 5, 6])),
            ),
            (TeamAllowlist::All, TeamAllowlist::All),
        ] {
            let (access, mut poller) = TeamAccess::new(env.clone());
            poller.accept(rows(&read)).unwrap();
            let processing = access.processing_teams().unwrap();

            assert_eq!(processing.scope(), &expected_scope, "env {env:?}");
            for team_id in [2, 3, 5, 6, 7, 8] {
                let expected = expected_scope.includes(team_id);
                assert_eq!(
                    access.processes(team_id),
                    expected,
                    "env {env:?}, team {team_id}"
                );
                assert_eq!(
                    processing.scope().includes(team_id),
                    expected,
                    "env {env:?}, team {team_id}"
                );
            }
        }
    }

    #[test]
    fn a_read_that_lost_a_row_or_regranted_an_off_row_keeps_those_rows_and_applies_the_others() {
        let (access, mut poller) = TeamAccess::new(env_only(&[]));
        assert_eq!(poller.accept(rows(&[])).unwrap(), 0);
        assert!(access.is_loaded());
        let (accepted, set) = sets_last_success(|| {
            poller.accept(rows(&[(5, "off"), (6, "processing"), (7, "processing")]))
        });
        accepted.unwrap();
        assert!(set);

        let (regressed, set) = sets_last_success(|| {
            poller.accept(rows(&[(5, "processing"), (6, "off"), (8, "processing")]))
        });
        assert!(matches!(regressed, Err(PollError::RegressedRows(teams)) if teams == [5, 7]));
        assert!(!set, "a reader that keeps a regressed row must look stale");
        for (team_id, expected) in [(5, false), (6, false), (7, true), (8, true)] {
            assert_eq!(access.processes(team_id), expected, "team {team_id}");
        }
    }

    #[test]
    fn a_read_that_changes_granted_levels_or_turns_every_row_off_is_accepted() {
        let (access, mut poller) = TeamAccess::new(env_only(&[]));
        poller
            .accept(rows(&[(5, "processing"), (6, "serving")]))
            .unwrap();
        poller
            .accept(rows(&[(5, "serving"), (6, "processing")]))
            .unwrap();

        assert_eq!(poller.accept(rows(&[(5, "off"), (6, "off")])).unwrap(), 0);
        assert!(!access.processes(5));
        assert!(!access.processes(6));
    }

    #[test]
    fn an_unknown_level_fails_the_read() {
        let parsed =
            AccessRows::parse(vec![(5, "processing".to_string()), (6, "beta".to_string())]);

        assert!(matches!(
            parsed,
            Err(PollError::UnknownLevel { team_id: 6, level }) if level == "beta"
        ));
    }
}