Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(cohorts): scope the realtime cohort catalog load to the team allowlist (#109373)

posthog[bot]8c449bd22b26
diff --git a/rust/cohort-core/src/filters/loader.rs b/rust/cohort-core/src/filters/loader.rs
index 0b2c6eb48afa..455bde140107 100644
--- a/rust/cohort-core/src/filters/loader.rs
+++ b/rust/cohort-core/src/filters/loader.rs
@@ -1,5 +1,4 @@
-//! SQL load + pure row→catalog transform. [`load_realtime_cohorts`] is the only DB touch;
-//! [`build_catalog_from_rows`] is pure so the catalog logic is testable without Postgres.
+//! SQL loads + pure row→catalog transform.
use std::collections::HashMap;
@@ -19,13 +18,23 @@ use crate::metrics::{
};
use crate::seed::{BehavioralShapeHash, PersonShapeHash, ScopeKind, ShapeHashError};
+macro_rules! realtime_cohorts_select {
+ () => {
+ "SELECT c.id, c.team_id, c.filters, \
+ c.behavioral_filters_shape_hash, c.person_filters_shape_hash, t.timezone \
+ FROM posthog_cohort c \
+ JOIN posthog_team t ON t.id = c.team_id \
+ WHERE c.cohort_type = 'realtime' AND c.deleted = false AND c.filters IS NOT NULL"
+ };
+}
+
/// Realtime cohorts to load, mirroring the Node filter manager's predicate, joined to
/// `posthog_team` for the team timezone the bucket variants use for calendar-day computation.
-pub const REALTIME_COHORTS_SQL: &str = "SELECT c.id, c.team_id, c.filters, \
- c.behavioral_filters_shape_hash, c.person_filters_shape_hash, t.timezone \
- FROM posthog_cohort c \
- JOIN posthog_team t ON t.id = c.team_id \
- WHERE c.cohort_type = 'realtime' AND c.deleted = false AND c.filters IS NOT NULL";
+pub const REALTIME_COHORTS_SQL: &str = realtime_cohorts_select!();
+
+/// Separate from [`REALTIME_COHORTS_SQL`] so a generic plan keeps the `team_id` index.
+const REALTIME_COHORTS_FOR_TEAMS_SQL: &str =
+ concat!(realtime_cohorts_select!(), " AND c.team_id = ANY($1)");
/// One realtime cohort row; `filters` is the `jsonb` column decoded to a `Value`.
#[derive(Debug, Clone, sqlx::FromRow)]
@@ -46,6 +55,17 @@ pub async fn load_realtime_cohorts(pool: &PgPool) -> Result<Vec<CohortRow>, Filt
Ok(rows)
}
+pub async fn load_realtime_cohorts_for_teams(
+ pool: &PgPool,
+ team_ids: &[i32],
+) -> Result<Vec<CohortRow>, FilterError> {
+ let rows = sqlx::query_as::<_, CohortRow>(REALTIME_COHORTS_FOR_TEAMS_SQL)
+ .bind(team_ids)
+ .fetch_all(pool)
+ .await?;
+ Ok(rows)
+}
+
/// Group rows by team into a catalog. A cohort that fails to parse is counted, warned, and skipped
/// rather than poisoning the rest of the catalog.
pub fn build_catalog_from_rows(rows: Vec<CohortRow>, cascade_enabled: bool) -> FilterCatalog {
diff --git a/rust/cohort-core/src/filters/mod.rs b/rust/cohort-core/src/filters/mod.rs
index 150646faa7cc..f075a44c24a9 100644
--- a/rust/cohort-core/src/filters/mod.rs
+++ b/rust/cohort-core/src/filters/mod.rs
@@ -9,7 +9,10 @@ use thiserror::Error;
pub use catalog::{FilterCatalog, Generation};
pub use leaf_classifier::{classify_leaf, LeafClass, LeafDropReason};
-pub use loader::{build_catalog_from_rows, load_realtime_cohorts, CohortRow, REALTIME_COHORTS_SQL};
+pub use loader::{
+ build_catalog_from_rows, load_realtime_cohorts, load_realtime_cohorts_for_teams, CohortRow,
+ REALTIME_COHORTS_SQL,
+};
pub use reverse_index::{BehavioralCandidates, LeafStateMeta, TeamFilters, TeamFiltersBuilder};
pub use tree::{
parse_cohort_tree, BehavioralLeafConfig, BehavioralValue, BoolOp, CohortLeaf,
diff --git a/rust/cohort-stream-processor/src/filters/manager.rs b/rust/cohort-stream-processor/src/filters/manager.rs
index 9fd067d9e805..99d70df2aa4d 100644
--- a/rust/cohort-stream-processor/src/filters/manager.rs
+++ b/rust/cohort-stream-processor/src/filters/manager.rs
@@ -18,7 +18,9 @@ use tokio::sync::Notify;
use tokio::task::JoinError;
use tracing::{debug, info, warn};
-use crate::filters::loader::{build_catalog_from_rows, load_realtime_cohorts, CohortRow};
+use crate::filters::loader::{
+ build_catalog_from_rows, load_realtime_cohorts, load_realtime_cohorts_for_teams, CohortRow,
+};
use crate::filters::FilterError;
use crate::filters::{FilterCatalog, Generation};
use crate::observability::metrics::{
@@ -202,14 +204,17 @@ impl CatalogHandle {
self.loaded_notify.notify_waiters();
}
- /// Query `posthog_cohort`, drop out-of-scope teams, rebuild the catalog, and swap it in.
+ /// Query `posthog_cohort` for the allowlisted teams, rebuild the catalog, and swap it in.
///
/// The staleness metrics are stamped here rather than in [`run_refresh_loop`] so the boot load
/// in `main` counts too — otherwise a pod that booted fine and then lost its refresh loop would
/// look identical to one that never loaded.
#[hotpath::measure(impl_type = "CatalogHandle")]
pub async fn refresh(&self, pool: &PgPool) -> Result<CatalogStats, CatalogRefreshError> {
- match self.refresh_from(|| load_realtime_cohorts(pool)).await {
+ match self
+ .refresh_from(|| load_allowlisted(pool, &self.allowlist))
+ .await
+ {
Ok(stats) => {
gauge!(FILTER_CATALOG_LAST_SUCCESS_TIMESTAMP_SECONDS).set(now_unix_seconds());
counter!(FILTER_CATALOG_REFRESH_TOTAL, "result" => "success").increment(1);
@@ -277,7 +282,20 @@ fn now_unix_seconds() -> f64 {
.unwrap_or(0.0)
}
-/// Drop rows outside the configured team scope before catalog construction.
+async fn load_allowlisted(
+ pool: &PgPool,
+ allowlist: &TeamAllowlist,
+) -> Result<Vec<CohortRow>, FilterError> {
+ match allowlist {
+ TeamAllowlist::All => load_realtime_cohorts(pool).await,
+ TeamAllowlist::Only(team_ids) => {
+ let team_ids: Vec<i32> = team_ids.iter().copied().collect();
+ load_realtime_cohorts_for_teams(pool, &team_ids).await
+ }
+ }
+}
+
+/// Backstop for the load query's team filter.
fn retain_allowlisted(rows: &mut Vec<CohortRow>, allowlist: &TeamAllowlist) {
rows.retain(|row| allowlist.includes(row.team_id));
}
@@ -354,7 +372,7 @@ fn next_interval(interval: Duration, jitter: Duration) -> Duration {
#[cfg(test)]
mod tests {
- use std::collections::HashSet;
+ use std::collections::{BTreeSet, HashSet};
use super::*;
use chrono_tz::UTC;
@@ -633,5 +651,27 @@ mod tests {
assert!(handle.is_loaded());
assert_eq!(handle.load().team_count(), stats.teams);
+
+ // Bypass `retain_allowlisted`, which would hide a missing SQL filter.
+ let keys = |rows: &[CohortRow]| -> BTreeSet<(i32, i32)> {
+ rows.iter().map(|row| (row.team_id, row.id)).collect()
+ };
+ let unscoped = load_allowlisted(&pool, &TeamAllowlist::All)
+ .await
+ .expect("unscoped load against live posthog_cohort");
+ if let Some(team_id) = unscoped.first().map(|row| row.team_id) {
+ let scoped = load_allowlisted(&pool, &TeamAllowlist::Only(HashSet::from([team_id])))
+ .await
+ .expect("team-scoped load against live posthog_cohort");
+ let expected: BTreeSet<_> = keys(&unscoped)
+ .into_iter()
+ .filter(|(team, _)| *team == team_id)
+ .collect();
+ assert_eq!(keys(&scoped), expected);
+ }
+ let none = load_allowlisted(&pool, &TeamAllowlist::Only(HashSet::new()))
+ .await
+ .expect("empty-scope load against live posthog_cohort");
+ assert!(none.is_empty());
}
}
diff --git a/rust/cohort-stream-processor/src/filters/mod.rs b/rust/cohort-stream-processor/src/filters/mod.rs
index 250ccb971d9f..699b01bdde35 100644
--- a/rust/cohort-stream-processor/src/filters/mod.rs
+++ b/rust/cohort-stream-processor/src/filters/mod.rs
@@ -3,7 +3,8 @@
pub mod manager;
pub use cohort_core::filters::{
- build_catalog_from_rows, load_realtime_cohorts, CohortRow, REALTIME_COHORTS_SQL,
+ build_catalog_from_rows, load_realtime_cohorts, load_realtime_cohorts_for_teams, CohortRow,
+ REALTIME_COHORTS_SQL,
};
pub use cohort_core::filters::{leaf_classifier, loader, reverse_index, tree};
pub use cohort_core::filters::{