Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(marketing-analytics): read active teams from query_log_archive (#121595)

Lucas Ricoyc43540b3e713
diff --git a/products/marketing_analytics/dags/marketing_precompute.py b/products/marketing_analytics/dags/marketing_precompute.py
index dfd3020f46b6..82d2e8b0096b 100644
--- a/products/marketing_analytics/dags/marketing_precompute.py
+++ b/products/marketing_analytics/dags/marketing_precompute.py
@@ -27,7 +27,7 @@ poisoning and no access-control bypass.
The audience is `MARKETING_PRECOMPUTE_TEAM_IDS`: comma-separated team IDs, empty to disable warming, or
`auto` to warm every team that has a conversion goal AND has opened marketing analytics recently
-(query_log). Unset, it warms the teams whose reads are precompute-only, on PostHog Cloud only: teams
+(query_log_archive). Unset, it warms the teams whose reads are precompute-only, on PostHog Cloud only: teams
with a conversion goal and the conversion read flag on, and recently active teams with a warehouse table
and the cost read flag on. `MARKETING_PRECOMPUTE_ACTIVE_DAYS` tunes the `auto` activity window.
"""
@@ -116,8 +116,7 @@ AUTO_AUDIENCE = "auto"
# Only keep teams warm while they are actually using marketing analytics. A team that has not opened it
# within this window drops out of the rolling warm set; its next visit reads not-ready and triggers a
# one-off background warm. This bounds the fleet to the active population instead of every team that ever
-# set a goal. query_log retention caps the effective lookback (~14 days), which is the low end of the range
-# we want anyway.
+# set a goal.
ACTIVE_DAYS_ENV_VAR = "MARKETING_PRECOMPUTE_ACTIVE_DAYS"
DEFAULT_ACTIVE_DAYS = 30
ACTIVE_TEAMS_QUERY_TIMEOUT_SECONDS = 60
@@ -155,21 +154,24 @@ MARKETING_PRECOMPUTE_TEAM_FAILED = Counter(
def _recently_active_team_ids(days: int) -> set[int] | None:
- """Teams that ran a marketing-analytics query within `days`, from query_log. None on failure.
+ """Teams that ran a marketing-analytics query within `days`, from query_log_archive. None on failure.
- None means "couldn't tell" (query_log unavailable / errored) and the caller skips the run. Warming
- every goal team instead would turn one transient failure into a fleet-wide backfill.
+ None means "couldn't tell" (query_log_archive unavailable / errored) and the caller skips the run.
+ Warming every goal team instead would turn one transient failure into a fleet-wide backfill.
"""
try:
# Match on the query_type tag in log_comment (the intended API, as the web warmer does), not a
# substring of the SQL text — the tag is stable, the annotation format is not.
+ # Read query_log_archive, not system.query_log: the archive stores the log_comment tags as typed
+ # lc_* columns and sorts by query start hour, so the window prunes by time. A scan of
+ # system.query_log on every replica parses log_comment per row and exceeds the timeout.
rows = sync_execute(
"""
- SELECT DISTINCT JSONExtractInt(log_comment, 'team_id') AS team_id
- FROM clusterAllReplicas(posthog, system, query_log)
+ SELECT DISTINCT team_id
+ FROM query_log_archive
WHERE type != 'QueryStart'
- AND event_time > now() - toIntervalDay(%(days)s)
- AND JSONExtractString(log_comment, 'query_type') IN (
+ AND initial_query_start_time > now() - toIntervalDay(%(days)s)
+ AND lc_query_type IN (
'marketing_analytics_table_query',
'marketing_analytics_aggregated_query'
)
@@ -210,7 +212,7 @@ def _costs_flag_team_ids() -> set[int]:
Cost reads never write, so the warmer is the only producer of cost rows. The two read flags roll out
independently, so a team with only the cost flag on must be in the audience too. Otherwise its cost
windows expire and every source falls back to the live union. Like `_read_flag_team_ids`, this records
- no exposure. When query_log is unavailable this returns no teams, so the conversion audience still warms.
+ no exposure. When query_log_archive is unavailable this returns no teams, so the conversion audience still warms.
"""
active_days = int(os.getenv(ACTIVE_DAYS_ENV_VAR, str(DEFAULT_ACTIVE_DAYS)))
active_team_ids = _recently_active_team_ids(active_days)
@@ -235,7 +237,7 @@ def get_selected_team_ids() -> list[int]:
kill switch): a comma-separated list with blank or invalid entries skipped.
`auto` warms every team that both has a conversion goal (`TeamMarketingAnalyticsConfig`) and has
- opened marketing analytics recently (query_log), which keeps the rolling warm set to the active
+ opened marketing analytics recently (query_log_archive), which keeps the rolling warm set to the active
population. Cold teams are warmed on-demand on their next visit instead.
"""
raw = os.getenv(SELECTED_TEAM_IDS_ENV_VAR)
diff --git a/products/marketing_analytics/dags/tests/test_marketing_precompute.py b/products/marketing_analytics/dags/tests/test_marketing_precompute.py
index 123b4df3abcf..b3f12202473b 100644
--- a/products/marketing_analytics/dags/tests/test_marketing_precompute.py
+++ b/products/marketing_analytics/dags/tests/test_marketing_precompute.py
@@ -1,14 +1,23 @@
import os
from datetime import UTC, datetime
+import pytest
from posthog.test.base import APIBaseTest
from unittest.mock import MagicMock, patch
+from django.test import SimpleTestCase
+
import dagster
from parameterized import parameterized
from posthog.schema import MarketingAnalyticsDrillDownLevel
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.query_log_archive import (
+ DISTRIBUTED_QUERY_LOG_ARCHIVE_OPS_TABLE_SQL,
+ SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE,
+ SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE_SQL,
+)
from posthog.dags.common import chunk_ranges
from posthog.models import Organization, Team
@@ -26,6 +35,7 @@ from products.marketing_analytics.dags.marketing_precompute import (
SELECTED_TEAM_IDS_ENV_VAR,
SHARDS_ENV_VAR,
TEAM_CONCURRENCY_ENV_VAR,
+ _recently_active_team_ids,
get_selected_team_ids,
marketing_precompute_job,
split_marketing_precompute_teams_op,
@@ -132,6 +142,37 @@ class TestGetSelectedTeamIds:
assert get_selected_team_ids() == []
[email protected]("clickhouse_database")
+class TestRecentlyActiveTeamIds(SimpleTestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ super().setUpClass()
+ sync_execute(SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE_SQL())
+ sync_execute(DISTRIBUTED_QUERY_LOG_ARCHIVE_OPS_TABLE_SQL())
+
+ def _archive_query(self, team_id: int, query_type: str, days_ago: int) -> None:
+ sync_execute(
+ f"""
+ INSERT INTO {SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE}
+ (initial_query_start_time, team_id, initial_query_id, type, lc_query_type)
+ VALUES (now64(6) - toIntervalDay(%(days_ago)s), %(team_id)s, generateUUIDv4(), 'QueryFinish', %(query_type)s)
+ """,
+ {"team_id": team_id, "query_type": query_type, "days_ago": days_ago},
+ )
+
+ def test_reads_marketing_query_types_within_window_from_archive(self) -> None:
+ self._archive_query(101, "marketing_analytics_table_query", days_ago=1)
+ self._archive_query(102, "marketing_analytics_aggregated_query", days_ago=29)
+ self._archive_query(103, "web_overview_query", days_ago=1)
+ self._archive_query(104, "marketing_analytics_table_query", days_ago=31)
+ self._archive_query(0, "marketing_analytics_table_query", days_ago=1)
+
+ active = _recently_active_team_ids(30)
+ assert active is not None
+ # The archive MV also records this test run's own queries, so compare only the seeded teams.
+ assert active & {0, 101, 102, 103, 104} == {101, 102}
+
+
class TestShardSplit:
@parameterized.expand(
[("one_shard", "1", 1), ("more_teams_than_shards", "3", 3), ("more_shards_than_teams", "8", 5)]