Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(web-analytics): read eager warm audience from query_log_archive (#121594)

Lucas Ricoy1ad6f84c64cb
diff --git a/products/web_analytics/dags/eager_web_analytics_precompute.py b/products/web_analytics/dags/eager_web_analytics_precompute.py
index f9a62d7329d2..e13f082b7901 100644
--- a/products/web_analytics/dags/eager_web_analytics_precompute.py
+++ b/products/web_analytics/dags/eager_web_analytics_precompute.py
@@ -211,22 +211,28 @@ def _fetch_active_wa_team_ids(limit: int, lookback_hours: int = 24) -> list[int]
from posthog.clickhouse.client import sync_execute # noqa: PLC0415 — keep dagster import path light
try:
+ # query_log_archive keeps one row per query chain with the log_comment tags as typed columns, and its
+ # sort key starts with the hour of initial_query_start_time, so this reads only the lookback window.
+ # The archive has no trigger column. Async dashboard tiles and stale revalidations run in
+ # process_query_task, which tags them kind='celery', but a person viewing the dashboard starts both.
+ # Warmers run in other tasks or in Dagster, so they fail this filter.
rows = sync_execute(
"""
- SELECT JSONExtractInt(log_comment, 'team_id') AS team_id, count() AS n
- FROM clusterAllReplicas(%(cluster)s, system, query_log)
- WHERE event_time > now() - INTERVAL %(hours)s HOUR
- AND type = 'QueryFinish' AND is_initial_query = 1
- AND JSONExtractString(log_comment, 'query_type') LIKE 'stats_table%%'
- AND JSONExtractString(log_comment, 'feature') != 'cache_warmup'
- AND JSONExtractString(log_comment, 'trigger') = ''
- AND JSONExtractString(log_comment, 'access_method') != 'personal_api_key'
+ SELECT team_id, count() AS n
+ FROM query_log_archive
+ WHERE initial_query_start_time > now() - INTERVAL %(hours)s HOUR
+ AND lc_query_type LIKE 'stats_table%%'
+ AND (lc_kind = 'request' OR (lc_kind = 'celery' AND lc_id = 'posthog.tasks.tasks.process_query_task'))
+ AND lc_feature != 'cache_warmup'
+ AND lc_access_method != 'personal_api_key'
+ AND type = 'QueryFinish' AND is_initial_query
AND team_id != 0
GROUP BY team_id
ORDER BY n DESC
LIMIT %(limit)s
+ SETTINGS skip_unavailable_shards=1
""",
- {"cluster": "posthog", "hours": lookback_hours, "limit": limit},
+ {"hours": lookback_hours, "limit": limit},
)
return [int(row[0]) for row in rows]
except Exception:
diff --git a/products/web_analytics/dags/tests/test_eager_web_analytics_precompute.py b/products/web_analytics/dags/tests/test_eager_web_analytics_precompute.py
index 6c2a2f5b7ad8..037120a05142 100644
--- a/products/web_analytics/dags/tests/test_eager_web_analytics_precompute.py
+++ b/products/web_analytics/dags/tests/test_eager_web_analytics_precompute.py
@@ -3,10 +3,10 @@ from contextlib import contextmanager
from datetime import timedelta
from typing import Any
-from posthog.test.base import APIBaseTest
+from posthog.test.base import APIBaseTest, ClickhouseTestMixin
from unittest.mock import Mock, patch
-from django.test import override_settings
+from django.test import SimpleTestCase, override_settings
from django.utils import timezone
import dagster
@@ -15,6 +15,8 @@ from structlog.testing import capture_logs
from posthog.schema import WebAnalyticsPreComputeStrategy, WebStatsBreakdown
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.query_log_archive import SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE
from posthog.hogql_queries.query_runner import ExecutionMode
from posthog.models import Organization, Team
@@ -22,6 +24,7 @@ from products.analytics_platform.backend.models.preaggregation_job import Preagg
from products.web_analytics.dags.eager_web_analytics_precompute import (
BASELINE_BREAKDOWNS,
BASELINE_WINDOW_DAYS,
+ _fetch_active_wa_team_ids,
_resolve_eager_audience,
_warm_baseline_for_team,
warm_eager_baseline_op,
@@ -117,6 +120,60 @@ class TestResolveEagerAudience:
assert diag == {"teams_configured": len(expected), "active_teams_pct": pct, "active_teams": expected_active}
+class TestFetchActiveWaTeamIds(ClickhouseTestMixin, SimpleTestCase):
+ _DASHBOARD_READ: dict[str, str | int] = {
+ "lc_query_type": "stats_table_paths",
+ "lc_kind": "request",
+ "lc_id": "",
+ "lc_feature": "query",
+ "lc_access_method": "",
+ "type": "QueryFinish",
+ "is_initial_query": 1,
+ "hours_ago": 1,
+ }
+
+ def setUp(self) -> None:
+ super().setUp()
+ sync_execute(f"TRUNCATE TABLE {SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE}")
+
+ def _archive_reads(self, team_id: int, count: int, **overrides: str | int) -> None:
+ row = {**self._DASHBOARD_READ, **overrides}
+ sync_execute(
+ f"""
+ INSERT INTO {SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE}
+ (initial_query_start_time, event_time_microseconds, team_id, initial_query_id, is_initial_query,
+ type, lc_query_type, lc_kind, lc_id, lc_feature, lc_access_method)
+ SELECT
+ now() - INTERVAL %(hours_ago)s HOUR, now() - INTERVAL %(hours_ago)s HOUR, %(team_id)s,
+ toString(generateUUIDv4()), %(is_initial_query)s, %(type)s, %(lc_query_type)s, %(lc_kind)s,
+ %(lc_id)s, %(lc_feature)s, %(lc_access_method)s
+ FROM numbers(%(count)s)
+ """,
+ {**row, "team_id": team_id, "count": count},
+ )
+
+ @parameterized.expand(
+ [
+ ("cache_warming_task", {"lc_kind": "celery", "lc_id": "posthog.caching.warming.warm_insight_cache_task"}),
+ ("dagster_warmer_without_kind", {"lc_kind": ""}),
+ ("not_a_stats_table_tile", {"lc_query_type": "web_overview_query"}),
+ ("cache_warmup_feature", {"lc_feature": "cache_warmup"}),
+ ("personal_api_key", {"lc_access_method": "personal_api_key"}),
+ ("failed_query", {"type": "ExceptionWhileProcessing"}),
+ ("non_initial_query", {"is_initial_query": 0}),
+ ("outside_lookback", {"hours_ago": 30}),
+ ]
+ )
+ def test_ranks_dashboard_reads_and_drops_other_traffic(self, _name: str, excluded: dict[str, str | int]) -> None:
+ self._archive_reads(team_id=101, count=3, lc_kind="celery", lc_id="posthog.tasks.tasks.process_query_task")
+ self._archive_reads(team_id=102, count=5)
+ self._archive_reads(team_id=0, count=10)
+ self._archive_reads(team_id=103, count=10, **excluded)
+
+ assert _fetch_active_wa_team_ids(limit=10, lookback_hours=24) == [102, 101]
+ assert _fetch_active_wa_team_ids(limit=1, lookback_hours=24) == [102]
+
+
@patch("products.web_analytics.dags.eager_web_analytics_precompute.is_cloud", return_value=True)
class TestWarmEagerBaselineOp(APIBaseTest):
"""Integration-shaped tests for the op. Query runners are patched so