:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(cohorts): scope the realtime cohort catalog load to the team allowlist (#109373)
posthog[bot]8c449bd22b26
diff --git a/rust/cohort-core/src/filters/loader.rs b/rust/cohort-core/src/filters/loader.rsindex 0b2c6eb48afa..455bde140107 100644--- a/rust/cohort-core/src/filters/loader.rs+++ b/rust/cohort-core/src/filters/loader.rs@@ -1,5 +1,4 @@-//! SQL load + pure row→catalog transform. [`load_realtime_cohorts`] is the only DB touch;-//! [`build_catalog_from_rows`] is pure so the catalog logic is testable without Postgres.+//! SQL loads + pure row→catalog transform.use std::collections::HashMap;@@ -19,13 +18,23 @@ use crate::metrics::{};use crate::seed::{BehavioralShapeHash, PersonShapeHash, ScopeKind, ShapeHashError};+macro_rules! realtime_cohorts_select {+ () => {+ "SELECT c.id, c.team_id, c.filters, \+ c.behavioral_filters_shape_hash, c.person_filters_shape_hash, t.timezone \+ FROM posthog_cohort c \+ JOIN posthog_team t ON t.id = c.team_id \+ WHERE c.cohort_type = 'realtime' AND c.deleted = false AND c.filters IS NOT NULL"+ };+}+/// Realtime cohorts to load, mirroring the Node filter manager's predicate, joined to/// `posthog_team` for the team timezone the bucket variants use for calendar-day computation.-pub const REALTIME_COHORTS_SQL: &str = "SELECT c.id, c.team_id, c.filters, \- c.behavioral_filters_shape_hash, c.person_filters_shape_hash, t.timezone \- FROM posthog_cohort c \- JOIN posthog_team t ON t.id = c.team_id \- WHERE c.cohort_type = 'realtime' AND c.deleted = false AND c.filters IS NOT NULL";+pub const REALTIME_COHORTS_SQL: &str = realtime_cohorts_select!();++/// Separate from [`REALTIME_COHORTS_SQL`] so a generic plan keeps the `team_id` index.+const REALTIME_COHORTS_FOR_TEAMS_SQL: &str =+ concat!(realtime_cohorts_select!(), " AND c.team_id = ANY($1)");/// One realtime cohort row; `filters` is the `jsonb` column decoded to a `Value`.#[derive(Debug, Clone, sqlx::FromRow)]@@ -46,6 +55,17 @@ pub async fn load_realtime_cohorts(pool: &PgPool) -> Result<Vec<CohortRow>, FiltOk(rows)}+pub async fn load_realtime_cohorts_for_teams(+ pool: &PgPool,+ team_ids: &[i32],+) -> Result<Vec<CohortRow>, FilterError> {+ let rows = sqlx::query_as::<_, CohortRow>(REALTIME_COHORTS_FOR_TEAMS_SQL)+ .bind(team_ids)+ .fetch_all(pool)+ .await?;+ Ok(rows)+}+/// Group rows by team into a catalog. A cohort that fails to parse is counted, warned, and skipped/// rather than poisoning the rest of the catalog.pub fn build_catalog_from_rows(rows: Vec<CohortRow>, cascade_enabled: bool) -> FilterCatalog {diff --git a/rust/cohort-core/src/filters/mod.rs b/rust/cohort-core/src/filters/mod.rsindex 150646faa7cc..f075a44c24a9 100644--- a/rust/cohort-core/src/filters/mod.rs+++ b/rust/cohort-core/src/filters/mod.rs@@ -9,7 +9,10 @@ use thiserror::Error;pub use catalog::{FilterCatalog, Generation};pub use leaf_classifier::{classify_leaf, LeafClass, LeafDropReason};-pub use loader::{build_catalog_from_rows, load_realtime_cohorts, CohortRow, REALTIME_COHORTS_SQL};+pub use loader::{+ build_catalog_from_rows, load_realtime_cohorts, load_realtime_cohorts_for_teams, CohortRow,+ REALTIME_COHORTS_SQL,+};pub use reverse_index::{BehavioralCandidates, LeafStateMeta, TeamFilters, TeamFiltersBuilder};pub use tree::{parse_cohort_tree, BehavioralLeafConfig, BehavioralValue, BoolOp, CohortLeaf,diff --git a/rust/cohort-stream-processor/src/filters/manager.rs b/rust/cohort-stream-processor/src/filters/manager.rsindex 9fd067d9e805..99d70df2aa4d 100644--- a/rust/cohort-stream-processor/src/filters/manager.rs+++ b/rust/cohort-stream-processor/src/filters/manager.rs@@ -18,7 +18,9 @@ use tokio::sync::Notify;use tokio::task::JoinError;use tracing::{debug, info, warn};-use crate::filters::loader::{build_catalog_from_rows, load_realtime_cohorts, CohortRow};+use crate::filters::loader::{+ build_catalog_from_rows, load_realtime_cohorts, load_realtime_cohorts_for_teams, CohortRow,+};use crate::filters::FilterError;use crate::filters::{FilterCatalog, Generation};use crate::observability::metrics::{@@ -202,14 +204,17 @@ impl CatalogHandle {self.loaded_notify.notify_waiters();}- /// Query `posthog_cohort`, drop out-of-scope teams, rebuild the catalog, and swap it in.+ /// Query `posthog_cohort` for the allowlisted teams, rebuild the catalog, and swap it in.////// The staleness metrics are stamped here rather than in [`run_refresh_loop`] so the boot load/// in `main` counts too — otherwise a pod that booted fine and then lost its refresh loop would/// look identical to one that never loaded.#[hotpath::measure(impl_type = "CatalogHandle")]pub async fn refresh(&self, pool: &PgPool) -> Result<CatalogStats, CatalogRefreshError> {- match self.refresh_from(|| load_realtime_cohorts(pool)).await {+ match self+ .refresh_from(|| load_allowlisted(pool, &self.allowlist))+ .await+ {Ok(stats) => {gauge!(FILTER_CATALOG_LAST_SUCCESS_TIMESTAMP_SECONDS).set(now_unix_seconds());counter!(FILTER_CATALOG_REFRESH_TOTAL, "result" => "success").increment(1);@@ -277,7 +282,20 @@ fn now_unix_seconds() -> f64 {.unwrap_or(0.0)}-/// Drop rows outside the configured team scope before catalog construction.+async fn load_allowlisted(+ pool: &PgPool,+ allowlist: &TeamAllowlist,+) -> Result<Vec<CohortRow>, FilterError> {+ match allowlist {+ TeamAllowlist::All => load_realtime_cohorts(pool).await,+ TeamAllowlist::Only(team_ids) => {+ let team_ids: Vec<i32> = team_ids.iter().copied().collect();+ load_realtime_cohorts_for_teams(pool, &team_ids).await+ }+ }+}++/// Backstop for the load query's team filter.fn retain_allowlisted(rows: &mut Vec<CohortRow>, allowlist: &TeamAllowlist) {rows.retain(|row| allowlist.includes(row.team_id));}@@ -354,7 +372,7 @@ fn next_interval(interval: Duration, jitter: Duration) -> Duration {#[cfg(test)]mod tests {- use std::collections::HashSet;+ use std::collections::{BTreeSet, HashSet};use super::*;use chrono_tz::UTC;@@ -633,5 +651,27 @@ mod tests {assert!(handle.is_loaded());assert_eq!(handle.load().team_count(), stats.teams);++ // Bypass `retain_allowlisted`, which would hide a missing SQL filter.+ let keys = |rows: &[CohortRow]| -> BTreeSet<(i32, i32)> {+ rows.iter().map(|row| (row.team_id, row.id)).collect()+ };+ let unscoped = load_allowlisted(&pool, &TeamAllowlist::All)+ .await+ .expect("unscoped load against live posthog_cohort");+ if let Some(team_id) = unscoped.first().map(|row| row.team_id) {+ let scoped = load_allowlisted(&pool, &TeamAllowlist::Only(HashSet::from([team_id])))+ .await+ .expect("team-scoped load against live posthog_cohort");+ let expected: BTreeSet<_> = keys(&unscoped)+ .into_iter()+ .filter(|(team, _)| *team == team_id)+ .collect();+ assert_eq!(keys(&scoped), expected);+ }+ let none = load_allowlisted(&pool, &TeamAllowlist::Only(HashSet::new()))+ .await+ .expect("empty-scope load against live posthog_cohort");+ assert!(none.is_empty());}}diff --git a/rust/cohort-stream-processor/src/filters/mod.rs b/rust/cohort-stream-processor/src/filters/mod.rsindex 250ccb971d9f..699b01bdde35 100644--- a/rust/cohort-stream-processor/src/filters/mod.rs+++ b/rust/cohort-stream-processor/src/filters/mod.rs@@ -3,7 +3,8 @@pub mod manager;pub use cohort_core::filters::{- build_catalog_from_rows, load_realtime_cohorts, CohortRow, REALTIME_COHORTS_SQL,+ build_catalog_from_rows, load_realtime_cohorts, load_realtime_cohorts_for_teams, CohortRow,+ REALTIME_COHORTS_SQL,};pub use cohort_core::filters::{leaf_classifier, loader, reverse_index, tree};pub use cohort_core::filters::{