:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(web-analytics): read eager warm audience from query_log_archive (#121594)
Lucas Ricoy1ad6f84c64cb
diff --git a/products/web_analytics/dags/eager_web_analytics_precompute.py b/products/web_analytics/dags/eager_web_analytics_precompute.pyindex f9a62d7329d2..e13f082b7901 100644--- a/products/web_analytics/dags/eager_web_analytics_precompute.py+++ b/products/web_analytics/dags/eager_web_analytics_precompute.py@@ -211,22 +211,28 @@ def _fetch_active_wa_team_ids(limit: int, lookback_hours: int = 24) -> list[int]from posthog.clickhouse.client import sync_execute # noqa: PLC0415 — keep dagster import path lighttry:+ # query_log_archive keeps one row per query chain with the log_comment tags as typed columns, and its+ # sort key starts with the hour of initial_query_start_time, so this reads only the lookback window.+ # The archive has no trigger column. Async dashboard tiles and stale revalidations run in+ # process_query_task, which tags them kind='celery', but a person viewing the dashboard starts both.+ # Warmers run in other tasks or in Dagster, so they fail this filter.rows = sync_execute("""- SELECT JSONExtractInt(log_comment, 'team_id') AS team_id, count() AS n- FROM clusterAllReplicas(%(cluster)s, system, query_log)- WHERE event_time > now() - INTERVAL %(hours)s HOUR- AND type = 'QueryFinish' AND is_initial_query = 1- AND JSONExtractString(log_comment, 'query_type') LIKE 'stats_table%%'- AND JSONExtractString(log_comment, 'feature') != 'cache_warmup'- AND JSONExtractString(log_comment, 'trigger') = ''- AND JSONExtractString(log_comment, 'access_method') != 'personal_api_key'+ SELECT team_id, count() AS n+ FROM query_log_archive+ WHERE initial_query_start_time > now() - INTERVAL %(hours)s HOUR+ AND lc_query_type LIKE 'stats_table%%'+ AND (lc_kind = 'request' OR (lc_kind = 'celery' AND lc_id = 'posthog.tasks.tasks.process_query_task'))+ AND lc_feature != 'cache_warmup'+ AND lc_access_method != 'personal_api_key'+ AND type = 'QueryFinish' AND is_initial_queryAND team_id != 0GROUP BY team_idORDER BY n DESCLIMIT %(limit)s+ SETTINGS skip_unavailable_shards=1""",- {"cluster": "posthog", "hours": lookback_hours, "limit": limit},+ {"hours": lookback_hours, "limit": limit},)return [int(row[0]) for row in rows]except Exception:diff --git a/products/web_analytics/dags/tests/test_eager_web_analytics_precompute.py b/products/web_analytics/dags/tests/test_eager_web_analytics_precompute.pyindex 6c2a2f5b7ad8..037120a05142 100644--- a/products/web_analytics/dags/tests/test_eager_web_analytics_precompute.py+++ b/products/web_analytics/dags/tests/test_eager_web_analytics_precompute.py@@ -3,10 +3,10 @@ from contextlib import contextmanagerfrom datetime import timedeltafrom typing import Any-from posthog.test.base import APIBaseTest+from posthog.test.base import APIBaseTest, ClickhouseTestMixinfrom unittest.mock import Mock, patch-from django.test import override_settings+from django.test import SimpleTestCase, override_settingsfrom django.utils import timezoneimport dagster@@ -15,6 +15,8 @@ from structlog.testing import capture_logsfrom posthog.schema import WebAnalyticsPreComputeStrategy, WebStatsBreakdown+from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.query_log_archive import SHARDED_QUERY_LOG_ARCHIVE_V3_TABLEfrom posthog.hogql_queries.query_runner import ExecutionModefrom posthog.models import Organization, Team@@ -22,6 +24,7 @@ from products.analytics_platform.backend.models.preaggregation_job import Preaggfrom products.web_analytics.dags.eager_web_analytics_precompute import (BASELINE_BREAKDOWNS,BASELINE_WINDOW_DAYS,+ _fetch_active_wa_team_ids,_resolve_eager_audience,_warm_baseline_for_team,warm_eager_baseline_op,@@ -117,6 +120,60 @@ class TestResolveEagerAudience:assert diag == {"teams_configured": len(expected), "active_teams_pct": pct, "active_teams": expected_active}+class TestFetchActiveWaTeamIds(ClickhouseTestMixin, SimpleTestCase):+ _DASHBOARD_READ: dict[str, str | int] = {+ "lc_query_type": "stats_table_paths",+ "lc_kind": "request",+ "lc_id": "",+ "lc_feature": "query",+ "lc_access_method": "",+ "type": "QueryFinish",+ "is_initial_query": 1,+ "hours_ago": 1,+ }++ def setUp(self) -> None:+ super().setUp()+ sync_execute(f"TRUNCATE TABLE {SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE}")++ def _archive_reads(self, team_id: int, count: int, **overrides: str | int) -> None:+ row = {**self._DASHBOARD_READ, **overrides}+ sync_execute(+ f"""+ INSERT INTO {SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE}+ (initial_query_start_time, event_time_microseconds, team_id, initial_query_id, is_initial_query,+ type, lc_query_type, lc_kind, lc_id, lc_feature, lc_access_method)+ SELECT+ now() - INTERVAL %(hours_ago)s HOUR, now() - INTERVAL %(hours_ago)s HOUR, %(team_id)s,+ toString(generateUUIDv4()), %(is_initial_query)s, %(type)s, %(lc_query_type)s, %(lc_kind)s,+ %(lc_id)s, %(lc_feature)s, %(lc_access_method)s+ FROM numbers(%(count)s)+ """,+ {**row, "team_id": team_id, "count": count},+ )++ @parameterized.expand(+ [+ ("cache_warming_task", {"lc_kind": "celery", "lc_id": "posthog.caching.warming.warm_insight_cache_task"}),+ ("dagster_warmer_without_kind", {"lc_kind": ""}),+ ("not_a_stats_table_tile", {"lc_query_type": "web_overview_query"}),+ ("cache_warmup_feature", {"lc_feature": "cache_warmup"}),+ ("personal_api_key", {"lc_access_method": "personal_api_key"}),+ ("failed_query", {"type": "ExceptionWhileProcessing"}),+ ("non_initial_query", {"is_initial_query": 0}),+ ("outside_lookback", {"hours_ago": 30}),+ ]+ )+ def test_ranks_dashboard_reads_and_drops_other_traffic(self, _name: str, excluded: dict[str, str | int]) -> None:+ self._archive_reads(team_id=101, count=3, lc_kind="celery", lc_id="posthog.tasks.tasks.process_query_task")+ self._archive_reads(team_id=102, count=5)+ self._archive_reads(team_id=0, count=10)+ self._archive_reads(team_id=103, count=10, **excluded)++ assert _fetch_active_wa_team_ids(limit=10, lookback_hours=24) == [102, 101]+ assert _fetch_active_wa_team_ids(limit=1, lookback_hours=24) == [102]++@patch("products.web_analytics.dags.eager_web_analytics_precompute.is_cloud", return_value=True)class TestWarmEagerBaselineOp(APIBaseTest):"""Integration-shaped tests for the op. Query runners are patched so