Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(marketing-analytics): warm cost precompute for teams with only the cost flag (#109072)

posthog[bot]a0b9f3cd4955
diff --git a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py
index 680713935d9a..5d2484bbbc2f 100644
--- a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py
+++ b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py
@@ -188,6 +188,8 @@ class MarketingAnalyticsBaseQueryRunner(AnalyticsQueryRunner[ResponseType], ABC,
self._costs_precompute_used: bool = False
self._costs_sources_materialized: int = 0
self._costs_grain: Optional[str] = None
+ # Why the cost read did or did not use the precompute, so a team stuck on the live union shows the cause.
+ self._costs_precompute_outcome: Optional[str] = None
# Without this, a rollout where every query falls back to the live path looks identical to
# one that works.
self._live_session_resolution_used: bool = False
@@ -257,6 +259,7 @@ class MarketingAnalyticsBaseQueryRunner(AnalyticsQueryRunner[ResponseType], ABC,
"costs_precompute_used": self._costs_precompute_used,
"costs_sources_materialized": self._costs_sources_materialized,
"costs_grain": self._costs_grain,
+ "costs_precompute_outcome": self._costs_precompute_outcome,
"live_session_resolution_used": self._live_session_resolution_used,
}
if error is None:
@@ -396,6 +399,7 @@ class MarketingAnalyticsBaseQueryRunner(AnalyticsQueryRunner[ResponseType], ABC,
mat_adapters = [a for a in mat_adapters if a.supports_level(grain)]
grain_value = str(grain.value)
if not mat_adapters:
+ self._costs_precompute_outcome = "fallback_no_materializable_sources"
logger.info(
"marketing_costs_precompute",
outcome="fallback_no_materializable_sources",
@@ -452,6 +456,7 @@ class MarketingAnalyticsBaseQueryRunner(AnalyticsQueryRunner[ResponseType], ABC,
if not materialized_source_ids:
# Nothing materialized — let the caller read every source live, as before.
+ self._costs_precompute_outcome = "fallback_no_jobs"
logger.info(
"marketing_costs_precompute",
outcome="fallback_no_jobs",
@@ -471,6 +476,7 @@ class MarketingAnalyticsBaseQueryRunner(AnalyticsQueryRunner[ResponseType], ABC,
self._costs_precompute_used = True
self._costs_sources_materialized = len(mat_adapters) - len(s3_fallback_adapters)
self._costs_grain = grain_value
+ self._costs_precompute_outcome = "used"
logger.info(
"marketing_costs_precompute",
outcome="used",
@@ -1248,6 +1254,7 @@ class MarketingAnalyticsBaseQueryRunner(AnalyticsQueryRunner[ResponseType], ABC,
union_subquery = self._build_costs_from_precompute(self.query_date_range)
except Exception:
logger.exception("cost_precompute_failed", team_id=self.team.pk)
+ self._costs_precompute_outcome = "failed"
union_subquery = None
if union_subquery is None:
# Only the S3 fallback consumes the live adapters. When precompute serves the query
diff --git a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py
index c3d8780d950d..cd19d80d6b74 100644
--- a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py
+++ b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py
@@ -52,6 +52,8 @@ MULTI_TOUCH_MODES: frozenset[AttributionMode] = frozenset(
# Conversion-goal reads serve from the precompute for a team with this flag on.
CONVERSION_PRECOMPUTE_FLAG = "marketing-analytics-precomputation"
+# Cost reads serve from the precompute for a team with this flag on.
+COSTS_PRECOMPUTE_FLAG = "marketing-analytics-costs-precomputation"
# Mutable by design: `from_team` builds a default instance and then overwrites the
@@ -142,7 +144,7 @@ class MarketingAnalyticsConfig:
group_properties=group_properties,
),
"costs": feature_enabled_or_false(
- "marketing-analytics-costs-precomputation",
+ COSTS_PRECOMPUTE_FLAG,
str(team.uuid),
groups=groups,
group_properties=group_properties,
@@ -166,8 +168,17 @@ class MarketingAnalyticsConfig:
distort the flag's own rollout numbers, and it has no use for the cost and session flags that
`_precompute_flags` evaluates alongside the conversion one.
"""
+ return MarketingAnalyticsConfig._flag_enabled_without_exposure(CONVERSION_PRECOMPUTE_FLAG, team)
+
+ @staticmethod
+ def costs_precompute_enabled_without_exposure(team: "Team") -> bool:
+ """Evaluate only the cost read flag, and record no exposure for it. Same audience-scan use as above."""
+ return MarketingAnalyticsConfig._flag_enabled_without_exposure(COSTS_PRECOMPUTE_FLAG, team)
+
+ @staticmethod
+ def _flag_enabled_without_exposure(flag: str, team: "Team") -> bool:
return feature_enabled_or_false(
- CONVERSION_PRECOMPUTE_FLAG,
+ flag,
str(team.uuid),
groups={"organization": str(team.organization.id)},
group_properties={"organization": {"id": str(team.organization.id)}},
diff --git a/products/marketing_analytics/dags/marketing_precompute.py b/products/marketing_analytics/dags/marketing_precompute.py
index 71adaf77f827..dfd3020f46b6 100644
--- a/products/marketing_analytics/dags/marketing_precompute.py
+++ b/products/marketing_analytics/dags/marketing_precompute.py
@@ -27,8 +27,9 @@ poisoning and no access-control bypass.
The audience is `MARKETING_PRECOMPUTE_TEAM_IDS`: comma-separated team IDs, empty to disable warming, or
`auto` to warm every team that has a conversion goal AND has opened marketing analytics recently
-(query_log). Unset, it warms the teams with a conversion goal and the read flag on, on PostHog Cloud
-only. `MARKETING_PRECOMPUTE_ACTIVE_DAYS` tunes the `auto` activity window.
+(query_log). Unset, it warms the teams whose reads are precompute-only, on PostHog Cloud only: teams
+with a conversion goal and the conversion read flag on, and recently active teams with a warehouse table
+and the cost read flag on. `MARKETING_PRECOMPUTE_ACTIVE_DAYS` tunes the `auto` activity window.
"""
import os
@@ -203,11 +204,33 @@ def _read_flag_team_ids() -> list[int]:
)
+def _costs_flag_team_ids() -> set[int]:
+ """Recently active teams with a warehouse table and the `marketing-analytics-costs-precomputation` flag on.
+
+ Cost reads never write, so the warmer is the only producer of cost rows. The two read flags roll out
+ independently, so a team with only the cost flag on must be in the audience too. Otherwise its cost
+ windows expire and every source falls back to the live union. Like `_read_flag_team_ids`, this records
+ no exposure. When query_log is unavailable this returns no teams, so the conversion audience still warms.
+ """
+ active_days = int(os.getenv(ACTIVE_DAYS_ENV_VAR, str(DEFAULT_ACTIVE_DAYS)))
+ active_team_ids = _recently_active_team_ids(active_days)
+ if not active_team_ids:
+ return set()
+ teams = (
+ Team.objects.filter(pk__in=active_team_ids)
+ .filter(pk__in=DataWarehouseTable.objects.filter(deleted=False).values("team_id"))
+ .select_related("organization")
+ )
+ return {team.pk for team in teams if MarketingAnalyticsConfig.costs_precompute_enabled_without_exposure(team)}
+
+
def get_selected_team_ids() -> list[int]:
"""Resolve which teams to warm.
- Unset, it warms every team that has a conversion goal and the `marketing-analytics-precomputation`
- read flag on. Those reads are precompute-only, so a flagged team the warmer skips reads not-ready.
+ Unset, it warms every team whose reads are precompute-only: a team with a conversion goal and the
+ `marketing-analytics-precomputation` read flag on, and a recently active team with a warehouse table and
+ the `marketing-analytics-costs-precomputation` flag on. A flagged team the warmer skips reads not-ready
+ for conversions, and reads costs from the live union.
Cloud only: self-hosted has no flag rollout to follow. Set, the env var wins (even when empty, as a
kill switch): a comma-separated list with blank or invalid entries skipped.
@@ -217,7 +240,7 @@ def get_selected_team_ids() -> list[int]:
"""
raw = os.getenv(SELECTED_TEAM_IDS_ENV_VAR)
if raw is None:
- return _read_flag_team_ids() if is_cloud() else []
+ return sorted(set(_read_flag_team_ids()) | _costs_flag_team_ids()) if is_cloud() else []
if raw.strip().lower() != AUTO_AUDIENCE:
return [int(part.strip()) for part in raw.split(",") if part.strip().isdigit()]
diff --git a/products/marketing_analytics/dags/tests/test_marketing_precompute.py b/products/marketing_analytics/dags/tests/test_marketing_precompute.py
index 5c5462fac0f9..123b4df3abcf 100644
--- a/products/marketing_analytics/dags/tests/test_marketing_precompute.py
+++ b/products/marketing_analytics/dags/tests/test_marketing_precompute.py
@@ -31,10 +31,12 @@ from products.marketing_analytics.dags.marketing_precompute import (
split_marketing_precompute_teams_op,
warm_selected_teams,
)
+from products.warehouse_sources.backend.facade.models import DataWarehouseTable
_ENSURE = "products.marketing_analytics.dags.marketing_precompute.ensure_precomputed"
_ACTIVE = "products.marketing_analytics.dags.marketing_precompute._recently_active_team_ids"
_IS_CLOUD = "products.marketing_analytics.dags.marketing_precompute.is_cloud"
+_SYNC_EXECUTE = "products.marketing_analytics.dags.marketing_precompute.sync_execute"
_FF = "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.feature_enabled_or_false"
_DB = "products.marketing_analytics.dags.marketing_precompute.Database"
_FACTORY = "products.marketing_analytics.dags.marketing_precompute.MarketingSourceFactory"
@@ -161,29 +163,45 @@ class TestConversionWarming(APIBaseTest):
return team
@parameterized.expand([("cloud", True), ("self_hosted", False)])
- def test_unset_audience_is_goal_teams_with_the_read_flag(self, _name, cloud):
- # Reads are precompute-only for flagged teams, so the default audience must follow the flag.
+ def test_unset_audience_is_teams_with_a_precompute_read_flag(self, _name, cloud):
+ # Reads are precompute-only for flagged teams, so the default audience must follow the flags. The
+ # conversion and cost flags roll out independently, and cost reads never write, so a team with only
+ # the cost flag on must be warmed too.
flagged = self._make_team("flagged", goals=[_PRECOMPUTABLE_GOAL])
- unflagged = self._make_team("unflagged", goals=[_PRECOMPUTABLE_GOAL])
+ self._make_team("unflagged", goals=[_PRECOMPUTABLE_GOAL])
flagged_no_goals = self._make_team("flagged_no_goals", goals=[])
- flagged_uuids = {str(flagged.uuid), str(flagged_no_goals.uuid)}
+ costs_only = self._make_team("costs_only", goals=[_PRECOMPUTABLE_GOAL])
+ costs_inactive = self._make_team("costs_inactive", goals=[])
+ costs_no_tables = self._make_team("costs_no_tables", goals=[])
+ costs_unflagged = self._make_team("costs_unflagged", goals=[])
+ for team in (costs_only, costs_inactive, costs_unflagged):
+ DataWarehouseTable.objects.create(
+ team=team, name="ads_stats", format="Parquet", url_pattern="https://example.com/ads/*.parquet"
+ )
+ conversion_uuids = {str(flagged.uuid), str(flagged_no_goals.uuid)}
+ costs_uuids = {str(costs_only.uuid), str(costs_inactive.uuid), str(costs_no_tables.uuid)}
def read_flag(flag, distinct_id, *args, **kwargs):
- return flag == "marketing-analytics-precomputation" and distinct_id in flagged_uuids
+ if flag == "marketing-analytics-precomputation":
+ return distinct_id in conversion_uuids
+ return flag == "marketing-analytics-costs-precomputation" and distinct_id in costs_uuids
with (
patch.dict(os.environ, {}, clear=False),
patch(_IS_CLOUD, return_value=cloud),
+ patch(_SYNC_EXECUTE, return_value=[(costs_only.pk,), (costs_no_tables.pk,), (costs_unflagged.pk,)]),
patch(_FF, side_effect=read_flag) as feature_enabled,
):
os.environ.pop(SELECTED_TEAM_IDS_ENV_VAR, None)
selected = get_selected_team_ids()
- assert selected == ([flagged.pk] if cloud else [])
- assert unflagged.pk not in selected
- # Picking an audience is not a read, so the scan evaluates the read flag alone and records no
- # exposure. Otherwise every goal team looks exposed to three flags, every hour.
- assert {call.args[0] for call in feature_enabled.call_args_list} <= {"marketing-analytics-precomputation"}
+ assert selected == (sorted([flagged.pk, costs_only.pk]) if cloud else [])
+ # Picking an audience is not a read, so the scan evaluates the read flags alone and records no
+ # exposure. Otherwise every candidate team looks exposed to every flag, every hour.
+ assert {call.args[0] for call in feature_enabled.call_args_list} <= {
+ "marketing-analytics-precomputation",
+ "marketing-analytics-costs-precomputation",
+ }
assert all(call.kwargs["send_feature_flag_events"] is False for call in feature_enabled.call_args_list)
def test_auto_audience_is_recently_active_teams_with_goals(self):