:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(marketing-analytics): read active teams from query_log_archive (#121595)
Lucas Ricoyc43540b3e713
diff --git a/products/marketing_analytics/dags/marketing_precompute.py b/products/marketing_analytics/dags/marketing_precompute.pyindex dfd3020f46b6..82d2e8b0096b 100644--- a/products/marketing_analytics/dags/marketing_precompute.py+++ b/products/marketing_analytics/dags/marketing_precompute.py@@ -27,7 +27,7 @@ poisoning and no access-control bypass.The audience is `MARKETING_PRECOMPUTE_TEAM_IDS`: comma-separated team IDs, empty to disable warming, or`auto` to warm every team that has a conversion goal AND has opened marketing analytics recently-(query_log). Unset, it warms the teams whose reads are precompute-only, on PostHog Cloud only: teams+(query_log_archive). Unset, it warms the teams whose reads are precompute-only, on PostHog Cloud only: teamswith a conversion goal and the conversion read flag on, and recently active teams with a warehouse tableand the cost read flag on. `MARKETING_PRECOMPUTE_ACTIVE_DAYS` tunes the `auto` activity window."""@@ -116,8 +116,7 @@ AUTO_AUDIENCE = "auto"# Only keep teams warm while they are actually using marketing analytics. A team that has not opened it# within this window drops out of the rolling warm set; its next visit reads not-ready and triggers a# one-off background warm. This bounds the fleet to the active population instead of every team that ever-# set a goal. query_log retention caps the effective lookback (~14 days), which is the low end of the range-# we want anyway.+# set a goal.ACTIVE_DAYS_ENV_VAR = "MARKETING_PRECOMPUTE_ACTIVE_DAYS"DEFAULT_ACTIVE_DAYS = 30ACTIVE_TEAMS_QUERY_TIMEOUT_SECONDS = 60@@ -155,21 +154,24 @@ MARKETING_PRECOMPUTE_TEAM_FAILED = Counter(def _recently_active_team_ids(days: int) -> set[int] | None:- """Teams that ran a marketing-analytics query within `days`, from query_log. None on failure.+ """Teams that ran a marketing-analytics query within `days`, from query_log_archive. None on failure.- None means "couldn't tell" (query_log unavailable / errored) and the caller skips the run. Warming- every goal team instead would turn one transient failure into a fleet-wide backfill.+ None means "couldn't tell" (query_log_archive unavailable / errored) and the caller skips the run.+ Warming every goal team instead would turn one transient failure into a fleet-wide backfill."""try:# Match on the query_type tag in log_comment (the intended API, as the web warmer does), not a# substring of the SQL text — the tag is stable, the annotation format is not.+ # Read query_log_archive, not system.query_log: the archive stores the log_comment tags as typed+ # lc_* columns and sorts by query start hour, so the window prunes by time. A scan of+ # system.query_log on every replica parses log_comment per row and exceeds the timeout.rows = sync_execute("""- SELECT DISTINCT JSONExtractInt(log_comment, 'team_id') AS team_id- FROM clusterAllReplicas(posthog, system, query_log)+ SELECT DISTINCT team_id+ FROM query_log_archiveWHERE type != 'QueryStart'- AND event_time > now() - toIntervalDay(%(days)s)- AND JSONExtractString(log_comment, 'query_type') IN (+ AND initial_query_start_time > now() - toIntervalDay(%(days)s)+ AND lc_query_type IN ('marketing_analytics_table_query','marketing_analytics_aggregated_query')@@ -210,7 +212,7 @@ def _costs_flag_team_ids() -> set[int]:Cost reads never write, so the warmer is the only producer of cost rows. The two read flags roll outindependently, so a team with only the cost flag on must be in the audience too. Otherwise its costwindows expire and every source falls back to the live union. Like `_read_flag_team_ids`, this records- no exposure. When query_log is unavailable this returns no teams, so the conversion audience still warms.+ no exposure. When query_log_archive is unavailable this returns no teams, so the conversion audience still warms."""active_days = int(os.getenv(ACTIVE_DAYS_ENV_VAR, str(DEFAULT_ACTIVE_DAYS)))active_team_ids = _recently_active_team_ids(active_days)@@ -235,7 +237,7 @@ def get_selected_team_ids() -> list[int]:kill switch): a comma-separated list with blank or invalid entries skipped.`auto` warms every team that both has a conversion goal (`TeamMarketingAnalyticsConfig`) and has- opened marketing analytics recently (query_log), which keeps the rolling warm set to the active+ opened marketing analytics recently (query_log_archive), which keeps the rolling warm set to the activepopulation. Cold teams are warmed on-demand on their next visit instead."""raw = os.getenv(SELECTED_TEAM_IDS_ENV_VAR)diff --git a/products/marketing_analytics/dags/tests/test_marketing_precompute.py b/products/marketing_analytics/dags/tests/test_marketing_precompute.pyindex 123b4df3abcf..b3f12202473b 100644--- a/products/marketing_analytics/dags/tests/test_marketing_precompute.py+++ b/products/marketing_analytics/dags/tests/test_marketing_precompute.py@@ -1,14 +1,23 @@import osfrom datetime import UTC, datetime+import pytestfrom posthog.test.base import APIBaseTestfrom unittest.mock import MagicMock, patch+from django.test import SimpleTestCase+import dagsterfrom parameterized import parameterizedfrom posthog.schema import MarketingAnalyticsDrillDownLevel+from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.query_log_archive import (+ DISTRIBUTED_QUERY_LOG_ARCHIVE_OPS_TABLE_SQL,+ SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE,+ SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE_SQL,+)from posthog.dags.common import chunk_rangesfrom posthog.models import Organization, Team@@ -26,6 +35,7 @@ from products.marketing_analytics.dags.marketing_precompute import (SELECTED_TEAM_IDS_ENV_VAR,SHARDS_ENV_VAR,TEAM_CONCURRENCY_ENV_VAR,+ _recently_active_team_ids,get_selected_team_ids,marketing_precompute_job,split_marketing_precompute_teams_op,@@ -132,6 +142,37 @@ class TestGetSelectedTeamIds:assert get_selected_team_ids() == [][email protected]("clickhouse_database")+class TestRecentlyActiveTeamIds(SimpleTestCase):+ @classmethod+ def setUpClass(cls) -> None:+ super().setUpClass()+ sync_execute(SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE_SQL())+ sync_execute(DISTRIBUTED_QUERY_LOG_ARCHIVE_OPS_TABLE_SQL())++ def _archive_query(self, team_id: int, query_type: str, days_ago: int) -> None:+ sync_execute(+ f"""+ INSERT INTO {SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE}+ (initial_query_start_time, team_id, initial_query_id, type, lc_query_type)+ VALUES (now64(6) - toIntervalDay(%(days_ago)s), %(team_id)s, generateUUIDv4(), 'QueryFinish', %(query_type)s)+ """,+ {"team_id": team_id, "query_type": query_type, "days_ago": days_ago},+ )++ def test_reads_marketing_query_types_within_window_from_archive(self) -> None:+ self._archive_query(101, "marketing_analytics_table_query", days_ago=1)+ self._archive_query(102, "marketing_analytics_aggregated_query", days_ago=29)+ self._archive_query(103, "web_overview_query", days_ago=1)+ self._archive_query(104, "marketing_analytics_table_query", days_ago=31)+ self._archive_query(0, "marketing_analytics_table_query", days_ago=1)++ active = _recently_active_team_ids(30)+ assert active is not None+ # The archive MV also records this test run's own queries, so compare only the seeded teams.+ assert active & {0, 101, 102, 103, 104} == {101, 102}++class TestShardSplit:@parameterized.expand([("one_shard", "1", 1), ("more_teams_than_shards", "3", 3), ("more_shards_than_teams", "8", 5)]