:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(web-analytics): select warming demand from query_log_archive (#121612)
Lucas Ricoy8af0cdfb1021
diff --git a/products/web_analytics/dags/cache_warming.py b/products/web_analytics/dags/cache_warming.pyindex c810afad4994..8dbeb27065fc 100644--- a/products/web_analytics/dags/cache_warming.py+++ b/products/web_analytics/dags/cache_warming.py@@ -27,6 +27,7 @@ from posthog.hogql.constants import LimitContextfrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.client.connection import Workload+from posthog.clickhouse.query_log_archive import QUERY_LOG_ARCHIVE_DATA_TABLEfrom posthog.clickhouse.query_tagging import Feature, Product, reset_query_tags, tag_queries, tags_contextfrom posthog.dags.common import JobOwnersfrom posthog.event_usage import EventSource@@ -149,19 +150,120 @@ UNWARMABLE_QUERY_KINDS = ("WebVitalsQuery",)INTERNAL_QUERY_TYPE_SUFFIXES = ("_lazy_insert", "_preflight")_INTERNAL_QUERY_TYPE_FILTER = " OR ".join(f"endsWith(query_type, '{s}')" for s in INTERNAL_QUERY_TYPE_SUFFIXES)-# Request kind (top-level log_comment `kind`) excluded from demand selection.+# Request kind (top-level log_comment `kind`) excluded from the system.query_log demand selection.# Temporal-kind requests are batch/scheduled workflows that run web queries# across nearly every team; counting them made the selection reflect background# traffic rather than real dashboard usage. UI and personal-API-key requests are# kept — those are the reads warming is meant to keep fast.EXCLUDED_REQUEST_KIND = "temporal"+# What counts as demand on the query_log_archive path. The archive has no+# `trigger` column, so it can't exclude background replays by trigger. They are+# told apart by kind and task id instead: the warmer's own replays run from+# Dagster with no kind, stale revalidation and warmingV2 run as celery under+# their own task names, and batch workflows run as temporal. UI and+# personal-API-key reads are `request`. A UI read that executes asynchronously+# (a dashboard tile, a scene's async load) is also user demand, but the celery+# prerun tags it kind=celery/id=<task name> and process_query_task lets those+# tags win over the inherited request tags, so in the archive it is a celery+# row whose id is the async query task. Keep this in step with+# _fetch_active_wa_team_ids in eager_web_analytics_precompute.py.+DEMAND_REQUEST_KIND = "request"+ASYNC_QUERY_TASK_KIND = "celery"+ASYNC_QUERY_TASK_ID = "posthog.tasks.tasks.process_query_task"+# Read-bytes ceiling for the demand-selection scan. The 14-day fleet-wide-# query_log scan reads ~40 TiB, over the default cap, so it's raised — but to a-# finite value (~150 TiB, generous headroom for fleet growth) rather than 0, so-# the ClickHouse kill switch's overload cap still clamps it during an overload.+# system.query_log scan (the preset-lane path) reads ~40 TiB, over the default+# cap, so it's raised — but to a finite value (~150 TiB, generous headroom for+# fleet growth) rather than 0, so the ClickHouse kill switch's overload cap still+# clamps it during an overload._SELECTION_MAX_BYTES_TO_READ = 150 * 1024**4+# Demand-selection source fragments. Both sources yield the same row columns+# (team_id, query_json_raw, web_query_kind, query_type, request_kind, feature,+# request_id) plus the columns its own demand filter reads, so one aggregation+# serves both.+#+# query_log_archive holds one row per query chain, with the log_comment tags in+# typed lc_* columns. Its sort key starts with the chain's start hour, so the+# window filter prunes by primary key, and the LowCardinality lc_query__kind+# filter skips the query JSON of every non-web row. The table is an+# AggregatingMergeTree read without FINAL, so a chain can still be split across+# unmerged rows. is_initial_query keeps only the row that holds the initial query,+# which is the only row with lc_query filled, and counting distinct+# initial_query_id makes a duplicate row harmless.+_ARCHIVE_ROW_COLUMNS = """+ team_id,+ lc_query AS query_json_raw,+ -- From the query JSON, not lc_query__kind. lc_query__kind holds a+ -- wrapped source's kind (WebVitalsQuery wraps a TrendsQuery), so+ -- the lc_query__kind prefilter below drops any web kind with a+ -- non-web source. That is safe only while all such kinds are in+ -- UNWARMABLE_QUERY_KINDS.+ JSONExtractString(lc_query, 'kind') AS web_query_kind,+ lc_query_type AS query_type,+ lc_kind AS request_kind,+ lc_id AS request_task_id,+ lc_feature AS feature,+ initial_query_id AS request_id"""+_ARCHIVE_SOURCE = f"""+ FROM {QUERY_LOG_ARCHIVE_DATA_TABLE}+ WHERE+ initial_query_start_time >= now() - INTERVAL %(days)s DAY+ AND startsWith(lc_query__kind, %(kind_prefix)s)+ AND is_initial_query+ AND type = 'QueryFinish'+ AND query_kind = 'Select'"""+_ARCHIVE_DEMAND_FILTER = """(+ request_kind = %(demand_request_kind)s+ OR (request_kind = %(async_query_task_kind)s AND request_task_id = %(async_query_task_id)s)+ )"""++# system.query_log keeps the preset_id tag the archive drops, so the preset lane+# still reads it. Dagster connects to offline nodes, while the user traffic to+# replay lands on other replicas, hence clusterAllReplicas. Web rows are spread+# through every granule, so no row filter here reduces the read of the large+# log_comment column across the window.+_QUERY_LOG_ROW_COLUMNS = """+ JSONExtractInt(log_comment, 'team_id') AS team_id,+ JSONExtractRaw(log_comment, 'query') AS query_json_raw,+ -- aliased away from the native `query_kind` column so the PREWHERE+ -- below binds to the column (Select/Insert/…), not this JSON kind+ -- (WebOverviewQuery/…); with prefer_column_name_to_alias=0 a name+ -- collision would resolve `query_kind = 'Select'` against the alias+ -- and silently select nothing.+ JSONExtractString(log_comment, 'query', 'kind') AS web_query_kind,+ JSONExtractString(log_comment, 'query_type') AS query_type,+ JSONExtractString(log_comment, 'kind') AS request_kind,+ JSONExtractString(log_comment, 'feature') AS feature,+ JSONExtractString(log_comment, 'trigger') AS trigger,+ query_id AS request_id"""+_QUERY_LOG_SOURCE = f"""+ FROM clusterAllReplicas(%(cluster)s, system.query_log)+ -- Filter the cheap native columns first so the big log_comment String+ -- is read only for surviving rows. is_initial_query alone drops roughly+ -- nine-tenths of the window (the rest are distributed subqueries), and+ -- query_kind excludes the warmer's own INSERT replays without a JSON+ -- parse — every warmable web query executes as a Select.+ PREWHERE+ type = 'QueryFinish'+ AND is_initial_query+ AND query_kind = 'Select'+ WHERE+ event_date >= toDate(now() - INTERVAL %(days)s DAY)+ AND event_time >= now() - INTERVAL %(days)s DAY+ -- cheap substring prefilter on the query's own kind before any+ -- JSON extraction; a superset of the kind filter below, false+ -- positives re-checked. A bare '%%Web%%' also matches user agents+ -- (AppleWebKit) and keeps most rows.+ AND log_comment LIKE '%%"kind":"{WARMABLE_QUERY_KIND_PREFIX}%%'"""+# trigger/feature exclusions keep the warmer's own replays (and every other+# background warmer) out of the demand counts. Otherwise a once-warmed shape+# keeps itself hot forever. Temporal-kind requests (batch/scheduled workflows)+# run web queries across nearly every team. Left in, they dominated the selection+# and filled the cap with shapes no dashboard reader ever loads.+_QUERY_LOG_DEMAND_FILTER = "trigger NOT IN %(background_triggers)s AND request_kind != %(excluded_request_kind)s"+def maybe_opt_into_lazy_precompute(query_json: dict) -> dict:"""Opt a replayed query into the lazy precompute path.@@ -560,9 +662,12 @@ def queries_to_keep_fresh(f"in the last {days} days (cap {max_shapes} shapes, preset lane {'on' if preset_lane else 'off'}).")- # Selection reads system.query_log across the whole cluster: Dagster connects- # to offline nodes, while the user traffic we want to replay lands on other- # replicas. (metrics_query_log_mv only looks usable — its DDL in+ # Selection reads query_log_archive, which holds every query chain in the+ # window with the log_comment tags as typed columns. A scan of+ # system.query_log over the same window must read the large log_comment+ # column for every row, which can run past max_execution_time. The preset+ # lane needs the preset_id tag, which only system.query_log keeps, so that+ # path still scans it. (metrics_query_log_mv only looks usable — its DDL in# posthog/models/query_metrics/sql.py was never migrated, the table does not# exist in production.) Demand is grouped by the NORMALIZED shape — the query# JSON with the range-varying and non-shape fields stripped@@ -575,24 +680,24 @@ def queries_to_keep_fresh(# worst hit. The representative to replay is the most-demanded variant (argMax# below); its range is then deepened to the widest the shape actually needs# (its distinct ranges come back in observed_date_froms). Demand is counted as- # distinct query_ids so duplicated log rows for one request can't inflate it.+ # distinct query ids so duplicated log rows for one request can't inflate it.# The per-shape hash is# cityHash64 of the group key rather than normalizedQueryHash(query), which# would read the full `query` SQL-text column — the largest in query_log —# across the whole window purely for a logging id.#- # The scan spans the whole WEB_ANALYTICS_WARMING_DAYS window fleet-wide, which- # exceeds the default max_bytes_to_read, so the cap is raised to+ # The system.query_log scan spans the whole WEB_ANALYTICS_WARMING_DAYS window+ # fleet-wide, which exceeds the default max_bytes_to_read, so the cap is raised to# _SELECTION_MAX_BYTES_TO_READ — a finite value, not 0, so the ClickHouse# kill switch's overload byte cap still clamps it (min(kill_switch_cap, ours))# and the giant scan is refused rather than piled on during an overload. The# run is also bounded by max_execution_time and by the demand-selection cache# upstream, so this heavy scan happens on the cache TTL cadence, not every run.#- # trigger/feature exclusions keep the warmer's own replays — and every other- # background warmer — out of the demand counts, otherwise a once-warmed shape- # would keep itself hot forever. LIKE literals are %%-escaped because- # clickhouse_driver %-formats the query when params are passed.+ # Each source excludes background replays its own way (see+ # _ARCHIVE_DEMAND_FILTER and _QUERY_LOG_DEMAND_FILTER). LIKE literals+ # are %%-escaped because clickhouse_driver %-formats the query when params are+ # passed.# The preset lane's SQL is spliced in rather than always present, so with the lane off# the scan does no extra JSON extraction, carries no extra aggregate state, and keeps# the original HAVING — the kill switch costs the fleet nothing. Every fragment is@@ -638,6 +743,11 @@ def queries_to_keep_fresh(preset_order_by = ""max_shapes_per_team = MAX_SHAPES_PER_TEAM+ if preset_lane:+ row_columns, source, demand_filter = _QUERY_LOG_ROW_COLUMNS, _QUERY_LOG_SOURCE, _QUERY_LOG_DEMAND_FILTER+ else:+ row_columns, source, demand_filter = _ARCHIVE_ROW_COLUMNS, _ARCHIVE_SOURCE, _ARCHIVE_DEMAND_FILTER+# nosemgrep: clickhouse-fstring-param-audit (interpolations are module-level constants from hardcoded tuples, not user input; everything dynamic is parameterized)results = sync_execute(f"""@@ -668,11 +778,10 @@ def queries_to_keep_fresh(team_id,normalized_shape,query_json_raw,- uniqExact(query_id) AS variant_count+ uniqExact(request_id) AS variant_count{preset_variant_columns}FROM (SELECT- JSONExtractInt(log_comment, 'team_id') AS team_id,-- The shape key: every top-level query field except the range--- varying / non-shape ones the precompute namespace ignores, so-- date-range and compare variants of one shape group together@@ -683,40 +792,14 @@ def queries_to_keep_fresh(kv -> concat(kv.1, char(31), kv.2),arrayFilter(kv -> NOT has(%(shape_ignored_fields)s, kv.1),- JSONExtractKeysAndValuesRaw(JSONExtractRaw(log_comment, 'query'))+ JSONExtractKeysAndValuesRaw(query_json_raw)))),char(30)) AS normalized_shape,- -- aliased away from the native `query_kind` column so the PREWHERE- -- below binds to the column (Select/Insert/…), not this JSON kind- -- (WebOverviewQuery/…); with prefer_column_name_to_alias=0 a name- -- collision would resolve `query_kind = 'Select'` against the alias- -- and silently select nothing.- JSONExtractString(log_comment, 'query', 'kind') AS web_query_kind,- JSONExtractString(log_comment, 'query_type') AS query_type,- JSONExtractString(log_comment, 'trigger') AS trigger,- JSONExtractString(log_comment, 'feature') AS feature,- JSONExtractString(log_comment, 'kind') AS request_kind,- JSONExtractRaw(log_comment, 'query') AS query_json_raw,- query_id+ {row_columns}{preset_row_columns}- FROM clusterAllReplicas(%(cluster)s, system.query_log)- -- Filter the cheap native columns first so the big log_comment String- -- is read only for surviving rows. is_initial_query alone drops roughly- -- nine-tenths of the window (the rest are distributed subqueries), and- -- query_kind excludes the warmer's own INSERT replays without a JSON- -- parse — every warmable web query executes as a Select.- PREWHERE- type = 'QueryFinish'- AND is_initial_query- AND query_kind = 'Select'- WHERE- event_date >= toDate(now() - INTERVAL %(days)s DAY)- AND event_time >= now() - INTERVAL %(days)s DAY- -- cheap substring prefilter before any JSON extraction; a- -- superset of the kind filter below, false positives re-checked- AND log_comment LIKE '%%{WARMABLE_QUERY_KIND_PREFIX}%%'+ {source}) AS subWHEREteam_id != 0@@ -724,14 +807,10 @@ def queries_to_keep_fresh(AND startsWith(web_query_kind, %(kind_prefix)s)AND web_query_kind NOT IN %(unwarmable_kinds)sAND NOT ({_INTERNAL_QUERY_TYPE_FILTER})- AND trigger NOT IN %(background_triggers)sAND feature != %(cache_warmup_feature)s-- Demand should reflect real product usage, not background query- -- traffic. Temporal-kind requests (batch/scheduled workflows) run- -- web queries across nearly every team — left in, they dominated the- -- selection and filled the cap with shapes no dashboard reader ever- -- loads. UI and personal-API-key traffic are kept.- AND request_kind != %(excluded_request_kind)s+ -- traffic.+ AND {demand_filter}GROUP BYteam_id,normalized_shape,@@ -758,6 +837,9 @@ def queries_to_keep_fresh("background_triggers": tuple(BACKGROUND_WARMING_TRIGGERS | SHARED_BACKGROUND_WARMING_TRIGGERS),"cache_warmup_feature": Feature.CACHE_WARMUP.value,"excluded_request_kind": EXCLUDED_REQUEST_KIND,+ "demand_request_kind": DEMAND_REQUEST_KIND,+ "async_query_task_kind": ASYNC_QUERY_TASK_KIND,+ "async_query_task_id": ASYNC_QUERY_TASK_ID,},settings={"max_bytes_to_read": _SELECTION_MAX_BYTES_TO_READ, "max_execution_time": 600},)@@ -791,7 +873,7 @@ def queries_to_keep_fresh(return rows-# The demand selection scans terabytes of query_log fleet-wide, so its result is+# The demand selection scans the whole lookback window fleet-wide, so its result is# cached in Redis and shared across warming runs: the hourly warmer replays the# cached shape list and the scan only re-runs once the cache expires# (WEB_ANALYTICS_WARMING_SELECTION_TTL_SECONDS). This is what lets the lookback@@ -809,7 +891,7 @@ def queries_to_keep_fresh(# the selection query (new filter, different grouping) would otherwise keep replaying# a stale blob written by the old logic until its TTL expired. Bump the version# whenever the selection query changes so the new logic takes effect on deploy.-_WARMABLE_QUERIES_STORAGE_KEY = "web_analytics/warmable_queries/v6.json.gz"+_WARMABLE_QUERIES_STORAGE_KEY = "web_analytics/warmable_queries/v7.json.gz"def _read_cached_warmable_queries(diff --git a/products/web_analytics/dags/tests/test_cache_warming.py b/products/web_analytics/dags/tests/test_cache_warming.pyindex 616ce2fbe2dd..26064538029b 100644--- a/products/web_analytics/dags/tests/test_cache_warming.py+++ b/products/web_analytics/dags/tests/test_cache_warming.py@@ -5,7 +5,7 @@ from concurrent.futures import Futurefrom datetime import datetime, timedeltafrom types import SimpleNamespace-from posthog.test.base import BaseTest+from posthog.test.base import BaseTest, ClickhouseTestMixinfrom unittest.mock import MagicMock, call, patchfrom django.test import SimpleTestCase@@ -14,6 +14,8 @@ from django.utils.dateparse import parse_datetimeimport dagsterfrom parameterized import parameterized+from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.query_log_archive import SHARDED_QUERY_LOG_ARCHIVE_V3_TABLEfrom posthog.clickhouse.query_tagging import Feature, get_query_tags, reset_query_tags, tag_queriesfrom posthog.exceptions import ClickHouseAtCapacityfrom posthog.hogql_queries.query_runner import ExecutionMode@@ -516,18 +518,39 @@ class TestFleetQuerySelection(BaseTest):],)+ @parameterized.expand(+ [+ # The archive path drops preset ids, so only the preset lane may still pay+ # for the system.query_log scan.+ ("archive", False, "FROM query_log_archive", "system.query_log"),+ ("preset_lane_query_log", True, "system.query_log", "query_log_archive"),+ ]+ )+ @patch("products.web_analytics.dags.cache_warming._apply_preset_lane", side_effect=lambda _c, rows, _m: rows)@patch("products.web_analytics.dags.cache_warming.sync_execute", return_value=[])- def test_selection_sql_survives_driver_percent_formatting(self, mock_exec: MagicMock) -> None:+ def test_selection_reads_the_source_its_lane_needs(+ self,+ _name: str,+ preset_lane: bool,+ expected_source: str,+ unexpected_source: str,+ mock_exec: MagicMock,+ _mock_lane: MagicMock,+ ) -> None:# clickhouse_driver %-formats the query when params are passed, so literal# % (the LIKE prefilter) must be written as %%. A bare % would crash only# in production, because tests mock sync_execute away.- queries_to_keep_fresh(dagster.build_op_context(), days=2, minimum_query_count=10, max_shapes=100)+ queries_to_keep_fresh(+ dagster.build_op_context(), days=2, minimum_query_count=10, max_shapes=100, preset_lane=preset_lane+ )sql, params = mock_exec.call_args[0]rendered = sql % dict.fromkeys(params, "1") # what the driver's substitution does- self.assertIn("LIKE '%Web%'", rendered)- self.assertIn("system.query_log", rendered)+ self.assertIn(expected_source, rendered)+ self.assertNotIn(unexpected_source, rendered)+ if preset_lane:+ self.assertIn("""LIKE '%"kind":"Web%'""", rendered)@patch("products.web_analytics.dags.cache_warming._read_cached_warmable_queries", return_value=None)@patch("products.web_analytics.dags.cache_warming._write_cached_warmable_queries")@@ -569,6 +592,84 @@ class TestFleetQuerySelection(BaseTest):self.assertIsInstance(entry["normalized_query_hash"], int)+class TestFleetQuerySelectionFromArchive(ClickhouseTestMixin, BaseTest):+ def setUp(self) -> None:+ super().setUp()+ sync_execute(f"TRUNCATE TABLE {SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE}")++ def _insert_chain(+ self,+ chain_id: str,+ query: dict,+ lc_kind: str = "request",+ lc_id: str = "",+ status: str = "QueryFinish",+ query_type: str = "web_overview_query",+ ) -> None:+ sync_execute(+ f"""+ INSERT INTO {SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE}+ (initial_query_start_time, team_id, initial_query_id, is_initial_query, type,+ query_kind_by_root, lc_kind, lc_id, lc_feature, lc_query_type, lc_query__kind, lc_query)+ VALUES (now64(6), %(team_id)s, %(chain_id)s, 1, %(status)s,+ (1, 'Select'), %(lc_kind)s, %(lc_id)s, 'query', %(query_type)s, %(kind)s, %(lc_query)s)+ """,+ {+ "team_id": self.team.pk,+ "chain_id": chain_id,+ "status": status,+ "lc_kind": lc_kind,+ "lc_id": lc_id,+ "query_type": query_type,+ "kind": query["kind"],+ "lc_query": json.dumps(query),+ },+ )++ # The archive has no trigger tag, so background replays are told apart from user+ # reads by kind and celery task id. Any of them leaking in keeps a shape hot forever.+ @parameterized.expand(+ [+ ("dagster_warmer_without_kind", "", ""),+ (+ "stale_revalidation_task",+ "celery",+ "products.web_analytics.backend.tasks.lazy_precompute_revalidation.revalidate_web_analytics_precompute",+ ),+ ("insight_cache_warmer_task", "celery", "posthog.caching.warming.warm_insight_cache_task"),+ ("temporal_workflow", "temporal", ""),+ ]+ )+ def test_counts_user_reads_and_ignores_background_replays(+ self, _name: str, background_kind: str, background_task_id: str+ ) -> None:+ user_shape = {"kind": "WebOverviewQuery", "dateRange": {"date_from": "-7d"}, "properties": []}+ background_shape = {"kind": "WebStatsTableQuery", "breakdownBy": "Page", "dateRange": {"date_from": "-7d"}}+ self._insert_chain("user-sync", user_shape)+ # A dashboard tile computed asynchronously on a user's behalf lands in the archive as+ # celery with the async query task's id. Without it the shape misses the demand floor.+ self._insert_chain(+ "user-async",+ {**user_shape, "dateRange": {"date_from": "-30d"}},+ lc_kind="celery",+ lc_id="posthog.tasks.tasks.process_query_task",+ )+ self._insert_chain("failed", user_shape, status="ExceptionWhileProcessing")+ self._insert_chain("bucket-build", user_shape, query_type="web_overview_lazy_insert")+ for index in range(2):+ self._insert_chain(+ f"background-{index}", background_shape, lc_kind=background_kind, lc_id=background_task_id+ )++ result = queries_to_keep_fresh(dagster.build_op_context(), days=1, minimum_query_count=2, max_shapes=100)++ rows = [row for row in result if row["team_id"] == self.team.pk]+ self.assertEqual(len(rows), 1)+ self.assertEqual(rows[0]["query_json"]["kind"], "WebOverviewQuery")+ self.assertEqual(rows[0]["query_count"], 2)+ self.assertEqual(sorted(rows[0]["observed_date_froms"]), ["-30d", "-7d"])++class TestPresetLane(BaseTest):"""The preset lane selects shapes a team ran under a saved filter preset, below thedemand floor. Everything here guards the two things that keeps honest: only real