Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(hogql): export only the read keys of properties from subqueries (#127964)

Sandy Spicerdc3c84c963e4
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.md
index 64586dc61f23..b37879c78bdb 100644
--- a/docs/published/handbook/engineering/databases/hogql-python.md
+++ b/docs/published/handbook/engineering/databases/hogql-python.md
@@ -59,6 +59,13 @@ Circular dependencies fail with a query error. Soft saves skip column inference
Query preparation shares a 60-second cooperative deadline across parsing, resolution, transformations, and printing.
AST visitors check the deadline while walking; native parser calls are checked when they return, rather than interrupted mid-call.
+## Disabling projection optimization
+
+Set `HOGQL_PROJECTION_OPTIMIZATION_ENABLED` to `false` in Django admin's instance settings to disable unused-column pruning and exported event-property narrowing for every project.
+The setting defaults to `true` and overrides the `optimizeProjections` query modifier when disabled.
+Workers pick up the change within about a minute; it affects newly compiled queries, not running queries or cached results.
+Disabling these optimizations can increase query time and memory use. Set the instance setting back to `true` to restore the normal per-query behavior.
+
## Pattern matching during query preparation
Expressions inside HogQL placeholders execute in the Python HogVM.
diff --git a/posthog/api/test/test_event.py b/posthog/api/test/test_event.py
index 8d68740865bb..7964c0b4f458 100644
--- a/posthog/api/test/test_event.py
+++ b/posthog/api/test/test_event.py
@@ -114,10 +114,10 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
# personhog, not Postgres, so it's not in this count. Was 16 before passing team=team
# into get_restricted_properties_for_team, which lets is_property_access_control_enabled
# skip its per-call Team+organization lookup. +1 for the saved-expressions fetch in the
- # HogQL database build. +1 for the flag-cache TTL instance setting, cold-cache here but
- # TTL-cached per worker in production. +1 for the organization's flag_evaluations mode lookup in
- # the HogQL database build.
- with self.assertNumQueries(16):
+ # HogQL database build. +1 each for the flag-cache TTL and projection kill-switch instance
+ # settings, cold-cache here but TTL-cached per worker in production. +1 for the
+ # organization's flag_evaluations mode lookup in the HogQL database build.
+ with self.assertNumQueries(17):
response = self.client.get(f"/api/projects/{self.team.id}/events/?event=event_name").json()
assert response["results"][0]["event"] == "event_name"
@@ -152,10 +152,10 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
# Group-type-mapping is read via personhog, not Postgres, so it's not in this count.
# Was 24 before passing team=team into get_restricted_properties_for_team, which lets
# is_property_access_control_enabled skip its per-call Team+organization lookup.
- # +1 for the saved-expressions fetch in the HogQL database build. +1 for the flag-cache
- # TTL instance setting, cold-cache here but TTL-cached per worker in production. +1 for
- # the organization's flag_evaluations mode lookup in the HogQL database build.
- expected_queries = 23 if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else 24
+ # +1 for the saved-expressions fetch in the HogQL database build. +1 each for the flag-cache
+ # TTL and projection kill-switch instance settings, cold-cache here but TTL-cached per worker
+ # in production. +1 for the organization's flag_evaluations mode lookup in the HogQL database build.
+ expected_queries = 24 if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else 25
with self.assertNumQueries(expected_queries):
response = self.client.get(
diff --git a/posthog/api/test/test_person.py b/posthog/api/test/test_person.py
index 58073ec7a2c1..e6f24efdc9e0 100644
--- a/posthog/api/test/test_person.py
+++ b/posthog/api/test/test_person.py
@@ -2264,10 +2264,10 @@ class TestPerson(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):
# The property-access-control feature check reuses the request's already-loaded team,
# so listing persons no longer pays a per-request Team lookup (was 16). +1 for the
# saved-expressions fetch in the HogQL database build. +1 each for the shared-database
- # kill-switch and flag-cache TTL instance settings, cold-cache here but TTL-cached per
- # worker in production. +1 for the organization's flag_evaluations mode lookup in the HogQL
- # database build.
- with self.assertNumQueries(17):
+ # kill-switch, projection kill-switch and flag-cache TTL instance settings, cold-cache here
+ # but TTL-cached per worker in production. +1 for the organization's flag_evaluations mode
+ # lookup in the HogQL database build.
+ with self.assertNumQueries(18):
response = self.client.get("/api/person/?limit=10").json()
self.assertEqual(len(response["results"]), 9)
returned_ids += [x["distinct_ids"][0] for x in response["results"]]
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.md
index d597c5875249..5bd904e984ec 100644
--- a/posthog/hogql/ACCESS_CONTROL.md
+++ b/posthog/hogql/ACCESS_CONTROL.md
@@ -191,6 +191,7 @@ They're masked when the query is printed to ClickHouse SQL, so a restricted read
- **Explicit reads** (`properties.email`) are replaced with `NULL`, and the resolver refuses to back them with a materialized column — `ClickHousePropertyResolver` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.
- **Whole-blob reads** (`SELECT properties` or `SELECT *`) have the restricted keys stripped from the returned JSON via `JSONDropKeysPool(...)` — `ClickHousePrinter._maybe_apply_json_drop_keys()` in `posthog/hogql/printer/clickhouse.py`.
- **Unrestricted key reads** (`properties.$browser`, `JSONExtractString(properties, '$browser')`) read the stored String blob without `JSONDropKeysPool`, because dropping keys cannot change their value. A read keeps the mask when its key overlaps a restricted key by dot segment (`customer` against a restricted `customer.ssn`), or when the key is an integer position — `key_read_overlaps_restriction()` in `posthog/hogql/restricted_properties.py`.
+- **Narrowed exports.** A subquery whose `properties` column is only read by key exports a map of those keys instead of the blob — `posthog/hogql/transforms/exported_properties.py`. On the legacy table each map value extracts its key from the masked blob. On native event JSON the resolver declines a key that overlaps a restriction, so the subquery exports the masked document — `stored_key_json_text()` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.
Group restrictions retain their group type index, so a same-named property on another group type stays readable. The masking also applies to the Postgres-backed `system.groups.group_properties` field.
diff --git a/posthog/hogql/printer/utils.py b/posthog/hogql/printer/utils.py
index 579f964fa97c..cdd2adc2b1ac 100644
--- a/posthog/hogql/printer/utils.py
+++ b/posthog/hogql/printer/utils.py
@@ -335,11 +335,16 @@ def prepare_ast_for_printing(
context.modifiers.propertyGroupsMode = PropertyGroupsMode.OPTIMIZED
if context.modifiers.optimizeProjections:
- with context.timings.measure("projection_pushdown"):
- node = pushdown_projections(node, context)
- # Pushdown mutates SelectQueryType.columns, staling cached CTE tables. Drop them so a
- # wrongly pruned column fails loudly at compile time instead of emitting broken SQL.
- context.cte_database_table_cache.clear()
+ from posthog.models.instance_setting import ( # noqa: PLC0415 -- keeps the printer importable without Django
+ get_instance_setting,
+ )
+
+ if get_instance_setting("HOGQL_PROJECTION_OPTIMIZATION_ENABLED"):
+ with context.timings.measure("projection_pushdown"):
+ node = pushdown_projections(node, context, narrow_properties=dialect == "clickhouse")
+ # Pushdown mutates SelectQueryType.columns, staling cached CTE tables. Drop them so a
+ # wrongly pruned column fails loudly at compile time instead of emitting broken SQL.
+ context.cte_database_table_cache.clear()
if dialect == "clickhouse" and context.modifiers.materializeRepeatedCTEs:
from django.conf import (
diff --git a/posthog/hogql/transforms/clickhouse_property_resolution.py b/posthog/hogql/transforms/clickhouse_property_resolution.py
index 716d9ab463d8..4955e9f6155e 100644
--- a/posthog/hogql/transforms/clickhouse_property_resolution.py
+++ b/posthog/hogql/transforms/clickhouse_property_resolution.py
@@ -222,6 +222,21 @@ def resolve_json_subcolumn_source(
)
+def stored_key_json_text(field_type: ast.FieldType, key: str, context: HogQLContext) -> ast.Expr | None:
+ """The JSON text of `<blob>.<key>` on the JSON events table, read from the key's own stored column and serialized as
+ `JSONExtractRaw(document, key)` returns it from the rebuilt document. None when no single column holds what the
+ document shows: the document leaves `$feature_flags` out in favour of its `$feature/<key>` pairs, and a key that a
+ restriction touches reads the masked document as text. The read is the `PropertyAccess` that lowering builds, so
+ it gets no property type and the value pass resolves it like any other key read.
+ """
+ if key == "$feature_flags" or property_path_overlaps_restriction(key, field_type.table_type, context):
+ return None
+ read = ast.PropertyAccess(
+ expr=ast.Field(chain=[field_type.name], type=field_type), keys=[key], type=ast.StringType(nullable=True)
+ )
+ return ast.Call(name="toJSONString", args=[read], type=ast.StringType(nullable=True))
+
+
def resolve_property_group_source(
field_type: ast.FieldType, property_name: str, context: HogQLContext
) -> MaterializedPropertySource | None:
diff --git a/posthog/hogql/transforms/exported_properties.py b/posthog/hogql/transforms/exported_properties.py
new file mode 100644
index 000000000000..bd09722a9f3d
--- /dev/null
+++ b/posthog/hogql/transforms/exported_properties.py
@@ -0,0 +1,291 @@
+"""Exported properties narrowing.
+
+A select that exports the events `properties` blob to readers that only read keys of it exports a map of those keys
+instead, under the same name:
+
+ SELECT properties.email FROM (SELECT properties FROM events)
+ ->
+ SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties['email']), ''), 'null'), '^"|"$', '')
+ FROM (SELECT map('email', JSONExtractRaw(events.properties, 'email')) AS properties FROM events)
+
+On the native JSON events table the exported blob is a document rebuilt per row from every stored property, and the
+reader parses that text again; the map reads one subcolumn per key instead. The column keeps its name, so a select that
+passes it through to its own reader needs no change, and a reader looks a key up in the map where it extracted it from
+the blob before.
+
+The map holds each key's JSON text as `JSONExtractRaw(blob, key)` returned it. On the native table the property
+resolution pass supplies that text from the key's own stored column (`stored_key_json_text`), or declines when no single
+column holds what the rebuilt document shows, and a declined key keeps the blob. Neither is a property read: a property
+read gets the property-definition type and, on the legacy table, a materialized column, which reads an empty string as
+NULL and decodes escapes. The reader's own extract stays around the lookup, so a missing key still reads as `''` and
+then NULL.
+
+A blob stays a blob when any reader reads it whole (`SELECT properties`, `JSONHas(properties, k)`, a DISTINCT, GROUP BY
+or ORDER BY on the column, a bare alias reference), when the resolution pass declines a key a native reader asks for, or
+when any column that carries the same value cannot be narrowed: the columns a blob passes through and the branches of a
+UNION ALL form one group, decided together. The decision runs on the reads `ProjectionPushdownOptimizer` collected,
+after it pruned.
+"""
+
+from typing import TYPE_CHECKING
+
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.models import StringJSONDatabaseField
+from posthog.hogql.database.schema.events import EventsTable
+from posthog.hogql.database.schema.util.null_sentinels import null_if_sentinels
+from posthog.hogql.visitor import clone_expr
+
+if TYPE_CHECKING:
+ from posthog.hogql.transforms.projection_pushdown import ProjectionPushdownOptimizer, SubqueryNode
+
+_MAP_TYPE_NAME = "Map(String, Nullable(String))"
+
+
+def narrow_exported_properties(optimizer: "ProjectionPushdownOptimizer", context: HogQLContext) -> None:
+ """Mutates the optimized query in place."""
+ _ExportedPropertiesNarrower(optimizer, context).run()
+
+
+def document_key_extract(document: ast.Expr, chain: list[str | int]) -> ast.Expr:
+ """The key's raw JSON text with the quotes trimmed, as the printer renders a property read of a text document."""
+ extract = ast.Call(
+ name="JSONExtractRaw",
+ args=[document, *(ast.Constant(value=key) for key in chain)],
+ type=ast.StringType(nullable=False),
+ )
+ return ast.Call(
+ name="replaceRegexpAll",
+ args=[null_if_sentinels(extract), ast.Constant(value='^"|"$'), ast.Constant(value="")],
+ type=ast.StringType(nullable=True),
+ )
+
+
+def _map_type() -> ast.MapType:
+ return ast.MapType(key_type=ast.StringType(nullable=False), value_type=ast.StringType(nullable=True))
+
+
+def _key_json_text(source: ast.Field, key: str, context: HogQLContext) -> ast.Expr | None:
+ """The key's JSON text as `JSONExtractRaw(blob, key)` returns it, read from the events table (module docstring), or
+ None when the native table has no single stored column for it."""
+ if context.uses_new_events_schema():
+ # Deferred: the resolution pass imports the printer, which imports the pruner and so this module.
+ from posthog.hogql.transforms.clickhouse_property_resolution import stored_key_json_text # noqa: PLC0415
+
+ field_type = source.type
+ assert isinstance(field_type, ast.FieldType)
+ return stored_key_json_text(field_type, key, context)
+ return ast.Call(
+ name="JSONExtractRaw",
+ args=[clone_expr(source, clear_types=False), ast.Constant(value=key)],
+ type=ast.StringType(nullable=False),
+ )
+
+
+def _column_name(expr: ast.Expr) -> str | None:
+ if isinstance(expr, ast.Alias):
+ return expr.alias
+ if isinstance(expr, ast.Field) and expr.chain:
+ return str(expr.chain[-1])
+ return None
+
+
+def _unwrap_aliases(expr: ast.Expr) -> ast.Expr:
+ while isinstance(expr, ast.Alias):
+ expr = expr.expr
+ return expr
+
+
+class _ExportedPropertiesNarrower:
+ def __init__(self, optimizer: "ProjectionPushdownOptimizer", context: HogQLContext):
+ self.optimizer = optimizer
+ self.context = context
+ self._registered = {id(node) for node in optimizer.subquery_map.values()}
+
+ def run(self) -> None:
+ for group in self._column_groups():
+ if not all(self._allows_map(node, name) for node, name in group):
+ continue
+ for node, name in group:
+ self._rewrite_reads(node, name)
+ if isinstance(node, ast.SelectQuery) and self._events_source(node, name) is not None:
+ self._rewrite_export(node, name)
+
+ def _column_groups(self) -> list[list[tuple["SubqueryNode", str]]]:
+ """The columns that carry one value between them, so one decision covers them all: a column and the child
+ column it passes through, a set query's column and the same column of each branch."""
+ nodes = list({id(node): node for node in self.optimizer.subquery_map.values()}.values())
+ seen: set[tuple[int, str]] = set()
+ groups: list[list[tuple[SubqueryNode, str]]] = []
+ for node in nodes:
+ for name in self._column_names(node):
+ if (id(node), name) in seen:
+ continue
+ seen.add((id(node), name))
+ group: list[tuple[SubqueryNode, str]] = []
+ queue: list[tuple[SubqueryNode, str]] = [(node, name)]
+ while queue:
+ member = queue.pop()
+ group.append(member)
+ for neighbor in self._neighbors(*member):
+ key = (id(neighbor[0]), neighbor[1])
+ if key not in seen:
+ seen.add(key)
+ queue.append(neighbor)
+ groups.append(group)
+ return groups
+
+ def _neighbors(self, node: "SubqueryNode", name: str) -> list[tuple["SubqueryNode", str]]:
+ neighbors: list[tuple[SubqueryNode, str]] = []
+ if isinstance(node, ast.SelectSetQuery):
+ for query in self.optimizer.union_queries(node):
+ branch_name = self.optimizer.branch_column_name(node, query, name)
+ if branch_name is not None:
+ neighbors.append((query, branch_name))
+ else:
+ passthrough = self._passthrough(node, name)
+ if passthrough is not None:
+ neighbors.append(passthrough)
+ neighbors.extend(self.optimizer.passthrough_parents.get((id(node), name), {}).values())
+ set_query = self.optimizer.union_of.get(id(node))
+ if set_query is not None:
+ set_name = self.optimizer.set_column_name(set_query, node, name)
+ if set_name is not None:
+ neighbors.append((set_query, set_name))
+ return neighbors
+
+ def _column_names(self, node: "SubqueryNode") -> list[str]:
+ if isinstance(node, ast.SelectSetQuery):
+ return self._column_names(self.optimizer.union_queries(node)[0])
+ return [name for name in (_column_name(expr) for expr in node.select) if name is not None]
+
+ def _select_expr(self, node: ast.SelectQuery, name: str) -> ast.Expr | None:
+ return next((expr for expr in node.select if _column_name(expr) == name), None)
+
+ def _source_field(self, node: ast.SelectQuery, name: str) -> ast.Field | None:
+ expr = self._select_expr(node, name)
+ source = _unwrap_aliases(expr) if expr is not None else None
+ if isinstance(source, ast.Field) and isinstance(source.type, ast.FieldType):
+ return source
+ return None
+
+ def _passthrough(self, node: ast.SelectQuery, name: str) -> tuple["SubqueryNode", str] | None:
+ source = self._source_field(node, name)
+ if source is None:
+ return None
+ assert isinstance(source.type, ast.FieldType)
+ child = self.optimizer.child_for(source.type.table_type)
+ return (child, source.type.name) if child is not None else None
+
+ def _events_source(self, node: ast.SelectQuery, name: str) -> ast.Field | None:
+ source = self._source_field(node, name)
+ if source is None or not isinstance(source.type, ast.FieldType) or not self._is_events_properties(source.type):
+ return None
+ return source
+
+ def _allows_map(self, node: "SubqueryNode", name: str) -> bool:
+ """Whether this member of a column group can carry the map instead of the blob.
+
+ A select may only pass a `properties` column along, or export it from the events table, when nothing in it
+ consumes the column's value: no reader of the column itself (a bare field read, or a clause naming it by alias
+ or by position), no DISTINCT, GROUP BY or HAVING, which compare or group by every column or the named ones, no
+ set operation but UNION ALL, and the select is neither the query's output nor a recursive CTE. These are the
+ ways a ClickHouse select can use a column's value without a field reference, so the check lives here in one
+ place rather than in the pruner's read tracking.
+ """
+ reads = self.optimizer.reads.get((id(node), name))
+ if reads is not None and reads.whole:
+ return False
+ if isinstance(node, ast.SelectSetQuery):
+ return all(set_node.set_operator.startswith("UNION ALL") for set_node in node.subsequent_select_queries)
+ if id(node) in self.optimizer.roots or id(node) in self.optimizer.recursive_cte_exprs:
+ return False
+ if id(node) not in self._registered and id(node) not in self.optimizer.union_of:
+ # A select inside an expression: its columns are its output
+ return False
+ if node.distinct or node.group_by or node.group_by_mode is not None or node.having is not None:
+ return False
+ if id(node) in self.optimizer.strict_set_branches:
+ return False
+ expr = self._select_expr(node, name)
+ if expr is None:
+ return False
+ if any(expr is column for column in self.optimizer.positional_columns(node)):
+ return False
+ if expr.type is not None and id(expr.type) in self.optimizer.alias_referenced_types:
+ return False
+ if self._passthrough(node, name) is not None:
+ return True
+ source = self._events_source(node, name)
+ if source is None:
+ return False
+ keys = reads.keys if reads is not None else set()
+ return all(_key_json_text(source, key, self.context) is not None for key in keys)
+
+ def _is_events_properties(self, field_type: ast.FieldType) -> bool:
+ table_type = field_type.table_type
+ if isinstance(table_type, ast.TableAliasType):
+ table_type = table_type.table_type
+ if not isinstance(table_type, ast.TableType) or not isinstance(table_type.table, EventsTable):
+ return False
+ database_field = field_type.resolve_database_field(self.context)
+ return isinstance(database_field, StringJSONDatabaseField) and database_field.name == "properties"
+
+ def _rewrite_reads(self, node: "SubqueryNode", name: str) -> None:
+ reads = self.optimizer.reads.get((id(node), name))
+ if reads is not None:
+ for alias_node, field_node in reads.sites.values():
+ self._rewrite_read(alias_node, field_node)
+
+ def _rewrite_read(self, alias_node: ast.Alias, field_node: ast.Field) -> None:
+ property_type = field_node.type
+ assert isinstance(property_type, ast.PropertyType)
+ base_chain = field_node.chain[: len(field_node.chain) - len(property_type.chain)]
+ base = ast.Field(chain=list(base_chain) or [property_type.field_type.name], type=property_type.field_type)
+ lookup = ast.ArrayAccess(
+ array=base, property=ast.Constant(value=property_type.chain[0]), type=ast.StringType(nullable=True)
+ )
+ new_expr = document_key_extract(lookup, list(property_type.chain[1:]))
+ alias_node.expr = new_expr
+ if isinstance(alias_node.type, ast.FieldAliasType):
+ # The resolver shares this type object with the select scope's column and alias maps
+ alias_node.type.type = new_expr.type or ast.UnknownType()
+
+ def _rewrite_export(self, node: ast.SelectQuery, name: str) -> None:
+ expr = self._select_expr(node, name)
+ source = self._events_source(node, name)
+ assert expr is not None and source is not None
+ reads = self.optimizer.reads.get((id(node), name))
+ map_expr = self._map_expr(source, sorted(reads.keys) if reads is not None else [])
+ innermost: ast.Alias | None = None
+ current = expr
+ while isinstance(current, ast.Alias):
+ innermost = current
+ current = current.expr
+ if innermost is None:
+ replacement = ast.Alias(
+ alias=name,
+ expr=map_expr,
+ hidden=True,
+ type=ast.FieldAliasType(alias=name, type=_map_type()),
+ )
+ node.select[node.select.index(expr)] = replacement
+ if isinstance(node.type, ast.SelectQueryType):
+ node.type.columns[name] = replacement.type or ast.UnknownType()
+ return
+ innermost.expr = map_expr
+ if isinstance(innermost.type, ast.FieldAliasType):
+ innermost.type.type = _map_type()
+
+ def _map_expr(self, source: ast.Field, keys: list[str]) -> ast.Expr:
+ args: list[ast.Expr] = []
+ for key in keys:
+ text = _key_json_text(source, key, self.context)
+ assert text is not None
+ args.append(ast.Constant(value=key))
+ args.append(text)
+ map_call = ast.Call(name="map", args=args, type=_map_type())
+ if keys:
+ return map_call
+ # Nobody reads a key; `map()` alone has the type Nothing, which a pass-through cannot carry
+ return ast.Call(name="accurateCast", args=[map_call, ast.Constant(value=_MAP_TYPE_NAME)], type=_map_type())
diff --git a/posthog/hogql/transforms/projection_pushdown.py b/posthog/hogql/transforms/projection_pushdown.py
index d8cff24f46e0..03afdfe34ae6 100644
--- a/posthog/hogql/transforms/projection_pushdown.py
+++ b/posthog/hogql/transforms/projection_pushdown.py
@@ -1,21 +1,44 @@
from collections import defaultdict
+from dataclasses import field
from typing import cast
from posthog.hogql import ast
from posthog.hogql.base import _T_AST
from posthog.hogql.context import HogQLContext
+from posthog.hogql.transforms.exported_properties import narrow_exported_properties
from posthog.hogql.visitor import TraversingVisitor
+from posthog.dataclasses import frozen
+
+SubqueryNode = ast.SelectQuery | ast.SelectSetQuery
+
+# Demand flows top-down, but a CTE's reader can be visited after the CTE, so one pass can leave a column's readers
+# incomplete. Each pass adds what the last one learned; the walk stops when a pass adds nothing. A query that still
+# changes after this many passes is pruned with what was collected and not narrowed.
+_MAX_COLLECTION_PASSES = 50
+
+
+@frozen(frozen=False)
+class ColumnReads:
+ """How the readers of one subquery column use it. `whole` is set by any read of the column itself; `keys` are the
+ first keys of `column.<key>` reads, with the reads themselves in `sites`, by the hidden alias the resolver wraps
+ every field in."""
+
+ whole: bool = False
+ keys: set[str] = field(default_factory=set)
+ sites: dict[int, tuple[ast.Alias, ast.Field]] = field(default_factory=dict)
+
class ProjectionPushdownOptimizer(TraversingVisitor):
"""
- Top-down projection pushdown optimizer that prunes unused asterisk-expanded columns from subqueries.
+ Top-down projection pushdown optimizer that prunes unused asterisk-expanded columns from subqueries, and records
+ how each subquery column is read so `exported_properties` can export only the keys read of a properties blob.
Algorithm Overview:
──────────────────
- This optimizer makes two top-down passes through the query tree: the first only collects
- demands, the second prunes with the complete demand set. A CTE can be consumed by nodes
- visited after it — notably sibling UNION branches — so pruning must wait for the full walk.
+ This optimizer makes several top-down passes through the query tree: the collecting passes only gather demands,
+ repeated until a pass learns nothing new, and the final pass prunes with the complete demand set. A CTE can be
+ consumed by nodes visited after it — notably sibling UNION branches — so pruning must wait for the full walk.
Each pass runs these phases per query:
@@ -23,21 +46,65 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
Phase 2 - Collect: Gather column demands from WHERE/GROUP BY/ORDER BY/etc
Phase 3 - Propagate: For demanded columns, visit their source to propagate to child queries
Phase 4 - Recurse: Visit child subqueries (repeat phases 1-4)
- Phase 5 - Prune: Remove unreferenced asterisk columns from this query (second pass only)
+ Phase 5 - Prune: Remove unreferenced asterisk columns from this query (final pass only)
+
+ A column that a query exports as it is (`SELECT properties FROM sub`) is not a read of it: how the column is used
+ is its own readers' business, so phase 3 hands their reads down to the child column instead. The reads recorded
+ here are the ones made through a field; whether a select's own shape (DISTINCT, a set operation, an ordinal, an
+ alias reference) consumes a column's value is `exported_properties`' question, answered from the select itself.
"""
- def __init__(self):
+ def __init__(self, context: HogQLContext | None = None):
super().__init__()
+ self.context = context
self.demands: dict[int, set[str]] = defaultdict(set)
- self.subquery_map: dict[int, ast.SelectQuery | ast.SelectSetQuery] = {}
- # True during the first pass; gates pruning to the second (see class docstring)
+ self.reads: dict[tuple[int, str], ColumnReads] = defaultdict(ColumnReads)
+ self.subquery_map: dict[int, SubqueryNode] = {}
+ # The select columns that export a child column as it is, by the child column they export
+ self.passthrough_parents: dict[tuple[int, str], dict[tuple[int, str], tuple[SubqueryNode, str]]] = defaultdict(
+ dict
+ )
+ self.roots: set[int] = set()
+ self.recursive_cte_exprs: set[int] = set()
+ self.converged: bool = False
+ self.union_of: dict[int, ast.SelectSetQuery] = {}
+ # Branches of a set query that compares rows (anything but UNION ALL)
+ self.strict_set_branches: set[int] = set()
+ # Select leaves of a set query one of whose leaves keeps every column, so the branches keep lining up
+ self.aligned_set_leaves: set[int] = set()
+ # True during the collecting passes; gates pruning to the final one (see class docstring)
self.collecting: bool = False
+ self._alias_stack: list[ast.Alias] = []
+ # Alias types a clause named directly (`ORDER BY p`), which reads the aliased column's value itself
+ self.alias_referenced_types: set[int] = set()
- def optimize(self, node: ast.SelectQuery | ast.SelectSetQuery) -> ast.SelectQuery | ast.SelectSetQuery:
+ def optimize(self, node: SubqueryNode) -> SubqueryNode:
+ self.roots = self._root_ids(node)
self.collecting = True
- self.visit(node)
+ signature: object = None
+ for _ in range(_MAX_COLLECTION_PASSES):
+ self.visit(node)
+ new_signature = self._demand_signature()
+ if new_signature == signature:
+ self.converged = True
+ break
+ signature = new_signature
self.collecting = False
- return cast(ast.SelectQuery | ast.SelectSetQuery, self.visit(node))
+ return cast(SubqueryNode, self.visit(node))
+
+ def _root_ids(self, node: SubqueryNode) -> set[int]:
+ if isinstance(node, ast.SelectSetQuery):
+ ids = {id(node)}
+ for query in self.union_queries(node):
+ ids |= self._root_ids(query)
+ return ids
+ return {id(node)}
+
+ def _demand_signature(self) -> object:
+ return (
+ tuple(sorted((key, tuple(sorted(names))) for key, names in self.demands.items())),
+ tuple(sorted((key, reads.whole, tuple(sorted(reads.keys))) for key, reads in self.reads.items())),
+ )
def visit_select_query(self, node: ast.SelectQuery) -> ast.SelectQuery:
# Phase 1: Register subqueries and CTEs for demand tracking
@@ -46,9 +113,16 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
if node.select_from:
self._register_subqueries(node.select_from)
- # Phase 2: Collect column demands from query clauses
+ # Phase 2: Collect column demands from query clauses. An asterisk column is demanded by the parent (phase 3),
+ # unless every column has to stay, when the child must keep it too.
+ keeps_all_columns = self.keeps_all_columns(node)
for expr in node.select:
- if not self._is_from_asterisk(expr):
+ if self._is_from_asterisk(expr) and not keeps_all_columns:
+ continue
+ passthrough = self._passthrough_source(expr)
+ if passthrough is not None:
+ self._demand_passthrough(node, expr, passthrough)
+ elif not self._is_from_asterisk(expr):
self.visit(expr)
if node.where:
@@ -60,6 +134,8 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
self.visit(expr)
if node.having:
self.visit(node.having)
+ if node.qualify:
+ self.visit(node.qualify)
if node.order_by:
for expr in node.order_by:
self.visit(expr)
@@ -67,6 +143,14 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
self.visit(node.limit)
if node.offset:
self.visit(node.offset)
+ if node.limit_by:
+ self.visit(node.limit_by)
+ if node.array_join_list:
+ for expr in node.array_join_list:
+ self.visit(expr)
+ if node.window_exprs:
+ for window_expr in node.window_exprs.values():
+ self.visit(window_expr)
if node.select_from:
self._collect_join_constraint_column_demands(node.select_from)
@@ -80,7 +164,7 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
if node.select_from:
self.visit(node.select_from)
- # Phase 5: Prune unreferenced asterisk columns from this query and CTEs (second pass only)
+ # Phase 5: Prune unreferenced asterisk columns from this query and CTEs (final pass only)
if not self.collecting:
self._prune_columns(node)
if node.ctes:
@@ -96,6 +180,15 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
1. Propagate parent demands to all branches uniformly
2. Visit each branch to apply pruning
"""
+ strict = any(not set_node.set_operator.startswith("UNION ALL") for set_node in node.subsequent_select_queries)
+ for query in self.union_queries(node):
+ self.union_of[id(query)] = node
+ if strict:
+ self.strict_set_branches.add(id(query))
+ leaves = self._select_leaves(node)
+ if any(self.keeps_all_columns(leaf) for leaf in leaves):
+ self.aligned_set_leaves.update(id(leaf) for leaf in leaves)
+
# Propagate parent demands to all branches uniformly
self._propagate_demands_to_union_branches(node)
@@ -106,6 +199,38 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
return node
+ def union_queries(self, node: ast.SelectSetQuery) -> list[SubqueryNode]:
+ return [node.initial_select_query, *(set_node.select_query for set_node in node.subsequent_select_queries)]
+
+ def _select_leaves(self, node: SubqueryNode) -> list[ast.SelectQuery]:
+ if isinstance(node, ast.SelectSetQuery):
+ return [leaf for query in self.union_queries(node) for leaf in self._select_leaves(query)]
+ return [node]
+
+ def _select_names(self, node: SubqueryNode) -> list[str | None]:
+ if isinstance(node, ast.SelectSetQuery):
+ return self._select_names(node.initial_select_query)
+ return [self._get_column_name(expr) for expr in node.select]
+
+ def branch_column_name(self, set_query: ast.SelectSetQuery, branch: SubqueryNode, name: str) -> str | None:
+ """The branch column that feeds the set query's column `name`. A set operation matches columns by position
+ and the first branch names the result; the resolver already reorders BY NAME branches to line up."""
+ set_names = self._select_names(set_query)
+ if name not in set_names:
+ return None
+ position = set_names.index(name)
+ branch_names = self._select_names(branch)
+ return branch_names[position] if position < len(branch_names) else None
+
+ def set_column_name(self, set_query: ast.SelectSetQuery, branch: SubqueryNode, branch_name: str) -> str | None:
+ """The set query's column that the branch column `branch_name` feeds"""
+ branch_names = self._select_names(branch)
+ if branch_name not in branch_names:
+ return None
+ position = branch_names.index(branch_name)
+ set_names = self._select_names(set_query)
+ return set_names[position] if position < len(set_names) else None
+
def _is_from_asterisk(self, expr: ast.Expr) -> bool:
"""Check if an expression was expanded from asterisk"""
if isinstance(expr, ast.Alias):
@@ -114,28 +239,144 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
return expr.from_asterisk
return False
+ def _passthrough_source(self, expr: ast.Expr) -> tuple[SubqueryNode, str] | None:
+ """The child query and column name behind a select column that exports a subquery or CTE column as it is."""
+ source = self._unwrap_aliases(expr)
+ if not isinstance(source, ast.Field) or not isinstance(source.type, ast.FieldType):
+ return None
+ child = self.child_for(source.type.table_type)
+ if child is None:
+ return None
+ return child, source.type.name
+
+ def _unwrap_aliases(self, expr: ast.Expr) -> ast.Expr:
+ while isinstance(expr, ast.Alias):
+ expr = expr.expr
+ return expr
+
+ def compares_rows(self, node: ast.SelectQuery) -> bool:
+ """Whether the select's rows are compared whole, so every column's value counts"""
+ return node.distinct or id(node) in self.strict_set_branches or node.group_by_mode == "all"
+
+ def positional_columns(self, node: ast.SelectQuery) -> list[ast.Expr]:
+ """The select columns a bare integer in ORDER BY, GROUP BY or LIMIT BY names by position (ClickHouse resolves
+ them positionally), so their values count and their positions must not move"""
+ referencing: list[ast.Expr] = [order.expr for order in node.order_by or []]
+ for group_expr in node.group_by or []:
+ if isinstance(group_expr, ast.GroupingSet):
+ referencing.extend(group_expr.exprs)
+ else:
+ referencing.append(group_expr)
+ if node.limit_by:
+ referencing.extend(node.limit_by.exprs)
+ columns: list[ast.Expr] = []
+ for expr in referencing:
+ if isinstance(expr, ast.Constant) and isinstance(expr.value, int) and not isinstance(expr.value, bool):
+ if 0 < expr.value <= len(node.select):
+ columns.append(node.select[expr.value - 1])
+ return columns
+
+ def keeps_all_columns(self, node: ast.SelectQuery) -> bool:
+ """Whether dropping a column would change the select's rows, move a position another clause names, or put the
+ select out of line with a sibling branch that keeps its columns"""
+ return self.compares_rows(node) or bool(self.positional_columns(node)) or id(node) in self.aligned_set_leaves
+
+ def _demand_passthrough(self, node: ast.SelectQuery, expr: ast.Expr, passthrough: tuple[SubqueryNode, str]) -> None:
+ child, child_name = passthrough
+ self.demands[id(child)].add(child_name)
+ col_name = self._get_column_name(expr)
+ if col_name is not None:
+ self.passthrough_parents[(id(child), child_name)][(id(node), col_name)] = (node, col_name)
+
def _propagate_demands_to_children(self, node: ast.SelectQuery) -> None:
"""
Propagate parent demands to child subqueries.
When a parent query demands a column from us, we need to visit that column's
- source expression to propagate the demand down to our child subqueries.
+ source expression to propagate the demand down to our child subqueries. A column
+ exported as it is hands the parent's reads of it down to the child column.
- If there are no parent demands but we have asterisk columns,
- this query defines the output. Visit all asterisk columns to preserve them.
+ A query without parent demands is either the query's output, which needs every
+ column as it is, or a subquery nobody reads a column of, which only has to keep
+ its rows.
"""
demanded_from_this = self.demands.get(id(node))
if not demanded_from_this:
- for expr in node.select:
- if self._is_from_asterisk(expr):
- self.visit(expr)
+ if id(node) in self.roots or id(node) not in self._registered_ids():
+ # The query's output, or a select inside an expression: every column is read as it is
+ for expr in node.select:
+ self._column_read_whole(node, expr)
+ elif not self.collecting:
+ # Only the final pass knows that nobody reads the column; a reader may still turn up while collecting
+ self._keep_unread_subquery_rows(node)
return
for col_name in demanded_from_this:
for expr in node.select:
- if self._get_column_name(expr) == col_name:
+ if self._get_column_name(expr) != col_name:
+ continue
+ passthrough = self._passthrough_source(expr)
+ if passthrough is None:
self.visit(expr)
break
+ child, child_name = passthrough
+ self._demand_passthrough(node, expr, passthrough)
+ parent_reads = self.reads.get((id(node), col_name))
+ if parent_reads is not None:
+ child_reads = self.reads[(id(child), child_name)]
+ child_reads.whole = child_reads.whole or parent_reads.whole
+ child_reads.keys.update(parent_reads.keys)
+ break
+
+ def _keep_unread_subquery_rows(self, node: ast.SelectQuery) -> None:
+ if self.keeps_all_columns(node) or id(node) in self.union_of:
+ # Every asterisk column stays: a row comparison counts them, and set query branches must line up
+ for expr in node.select:
+ if self._is_from_asterisk(expr):
+ self.visit(expr)
+ return
+ # Nobody reads a column, so the asterisk columns would only be computed and dropped. An explicit column keeps
+ # the rows; without one, the cheapest asterisk column does, and a JSON blob is the most expensive to compute.
+ explicit_columns = [expr for expr in node.select if not self._is_from_asterisk(expr)]
+ asterisk_columns = [expr for expr in node.select if self._is_from_asterisk(expr)]
+ if explicit_columns:
+ col_name = self._get_column_name(explicit_columns[0])
+ if col_name is not None:
+ self.demands[id(node)].add(col_name)
+ return
+ if not asterisk_columns:
+ return
+ keep = next((expr for expr in asterisk_columns if not self._is_json_blob(expr)), asterisk_columns[0])
+ col_name = self._get_column_name(keep)
+ if col_name is not None:
+ self.demands[id(node)].add(col_name)
+ passthrough = self._passthrough_source(keep)
+ if passthrough is not None:
+ self._demand_passthrough(node, keep, passthrough)
+ else:
+ self.visit(keep)
+
+ def _registered_ids(self) -> set[int]:
+ return {id(query) for query in self.subquery_map.values()}
+
+ def _column_read_whole(self, node: ast.SelectQuery, expr: ast.Expr) -> None:
+ """The column is the query's output, so the child column it passes through is read as it is"""
+ passthrough = self._passthrough_source(expr)
+ if passthrough is None:
+ self.visit(expr)
+ return
+ child, child_name = passthrough
+ self._demand_passthrough(node, expr, passthrough)
+ self.reads[(id(child), child_name)].whole = True
+
+ def _is_json_blob(self, expr: ast.Expr) -> bool:
+ source = self._unwrap_aliases(expr)
+ if not isinstance(source, ast.Field) or source.type is None or self.context is None:
+ return False
+ try:
+ return isinstance(source.type.resolve_constant_type(self.context), ast.StringJSONType)
+ except Exception:
+ return False
def _propagate_demands_to_union_branches(self, node: ast.SelectSetQuery) -> None:
"""
@@ -148,11 +389,18 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
if not demanded_from_this:
return
- all_queries = [node.initial_select_query] + [sn.select_query for sn in node.subsequent_select_queries]
-
- for query in all_queries:
- if isinstance(query, ast.SelectQuery | ast.SelectSetQuery):
- self.demands[id(query)].update(demanded_from_this)
+ for query in self.union_queries(node):
+ for col_name in demanded_from_this:
+ branch_name = self.branch_column_name(node, query, col_name)
+ if branch_name is None:
+ continue
+ self.demands[id(query)].add(branch_name)
+ parent_reads = self.reads.get((id(node), col_name))
+ if parent_reads is None:
+ continue
+ branch_reads = self.reads[(id(query), branch_name)]
+ branch_reads.whole = branch_reads.whole or parent_reads.whole
+ branch_reads.keys.update(parent_reads.keys)
def _register_subqueries(self, from_clause: ast.JoinExpr) -> None:
"""Register all subqueries in FROM clause before collecting demands"""
@@ -175,7 +423,10 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
"""Map type to subquery node for demand tracking"""
# Map both the type and the inner SelectQueryType if it's an alias
self.subquery_map[id(type_annotation)] = subquery
- if isinstance(type_annotation, ast.SelectQueryAliasType) and type_annotation.select_query_type:
+ if (
+ isinstance(type_annotation, ast.SelectQueryAliasType | ast.SelectViewType)
+ and type_annotation.select_query_type
+ ):
self.subquery_map[id(type_annotation.select_query_type)] = subquery
def _register_union_subquery(
@@ -187,7 +438,7 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
self.subquery_map[id(type_annotation)] = union_query
# Also register with the inner SelectSetQueryType for both aliased and non-aliased cases
- if isinstance(type_annotation, ast.SelectQueryAliasType):
+ if isinstance(type_annotation, ast.SelectQueryAliasType | ast.SelectViewType):
# Aliased: register with the inner select_query_type
if type_annotation.select_query_type:
self.subquery_map[id(type_annotation.select_query_type)] = union_query
@@ -197,43 +448,76 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
if type_annotation.types:
self.subquery_map[id(type_annotation.types[0])] = union_query
- def _get_subquery(self, table_type: ast.Type) -> ast.SelectQuery | ast.SelectSetQuery | None:
+ def _get_subquery(self, table_type: ast.Type) -> SubqueryNode | None:
"""Retrieve subquery by type"""
return self.subquery_map.get(id(table_type))
+ def child_for(self, table_type: ast.Type) -> SubqueryNode | None:
+ """The registered subquery, saved view body or CTE query a field's table type refers to, if any. The saved
+ view type has to map as well: otherwise nothing ever reads the view body and the final pass prunes it."""
+ if isinstance(
+ table_type, ast.SelectQueryType | ast.SelectQueryAliasType | ast.SelectSetQueryType | ast.SelectViewType
+ ):
+ return self._get_subquery(table_type)
+ if isinstance(table_type, ast.CTETableType | ast.CTETableAliasType):
+ # For CTE types, get the underlying SelectQueryType and demand from it
+ cte_select_query_type = (
+ table_type.cte_table_type.select_query_type
+ if isinstance(table_type, ast.CTETableAliasType)
+ else table_type.select_query_type
+ )
+ return self._get_subquery(cte_select_query_type)
+ return None
+
+ def visit_alias(self, node: ast.Alias) -> None:
+ self._alias_stack.append(node)
+ try:
+ super().visit_alias(node)
+ finally:
+ self._alias_stack.pop()
+
def visit_field(self, node: ast.Field) -> ast.Field:
"""Record demand when field references subquery or CTE column"""
field_type = node.type
+ if isinstance(field_type, ast.FieldAliasType):
+ self.alias_referenced_types.add(id(field_type))
+ return node
+
# A property read (`col.some_key`) still demands its base blob column. Its node type is a
# PropertyType wrapping the base FieldType, so unwrap to that base — otherwise the column is
# never recorded as demanded and gets pruned from an asterisk-expanded subquery/CTE, leaving a
# dangling reference that later blows up property lowering.
+ property_type: ast.PropertyType | None = None
if isinstance(field_type, ast.PropertyType):
+ property_type = field_type
field_type = field_type.field_type
# Handle both FieldType and ExpressionFieldType
if not isinstance(field_type, ast.FieldType | ast.ExpressionFieldType):
return node
- table_type = field_type.table_type
-
- if isinstance(table_type, ast.SelectQueryType | ast.SelectQueryAliasType | ast.SelectSetQueryType):
- subquery = self._get_subquery(table_type)
- if subquery:
- self.demands[id(subquery)].add(field_type.name)
- elif isinstance(table_type, ast.CTETableType | ast.CTETableAliasType):
- # For CTE types, get the underlying SelectQueryType and demand from it
- cte_select_query_type = (
- table_type.cte_table_type.select_query_type
- if isinstance(table_type, ast.CTETableAliasType)
- else table_type.select_query_type
- )
- cte_query = self._get_subquery(cte_select_query_type)
- if cte_query:
- self.demands[id(cte_query)].add(field_type.name)
-
+ child = self.child_for(field_type.table_type)
+ if child is None:
+ return node
+ self.demands[id(child)].add(field_type.name)
+ reads = self.reads[(id(child), field_type.name)]
+ if self._is_key_read(property_type, node):
+ assert property_type is not None
+ reads.keys.add(cast(str, property_type.chain[0]))
+ reads.sites[id(self._alias_stack[-1])] = (self._alias_stack[-1], node)
+ else:
+ reads.whole = True
return node
+ def _is_key_read(self, property_type: ast.PropertyType | None, node: ast.Field) -> bool:
+ """A `column.<key>` read the resolver wrapped in a hidden alias, which `exported_properties` can re-point.
+ A read of the JSON text itself, or of a position, needs the blob."""
+ if property_type is None or property_type.as_json or not property_type.chain:
+ return False
+ if not isinstance(property_type.chain[0], str):
+ return False
+ return bool(self._alias_stack) and self._alias_stack[-1].expr is node
+
def _collect_join_constraint_column_demands(self, from_clause: ast.JoinExpr) -> None:
"""Collect demands from JOIN constraints"""
if from_clause.constraint and from_clause.constraint.expr:
@@ -246,7 +530,7 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
def _prune_columns(self, node: ast.SelectQuery) -> None:
"""Prune asterisk-expanded columns that aren't demanded"""
demanded = self.demands.get(id(node))
- if not demanded:
+ if not demanded or self.keeps_all_columns(node):
return
pruned_select = []
@@ -292,10 +576,13 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
# Map the CTE's SelectQueryType to its expr for demand tracking
if cte.expr.type:
self.subquery_map[id(cte.expr.type)] = cte.expr
+ if cte.recursive:
+ self.recursive_cte_exprs.add(id(cte.expr))
def _visit_ctes(self, ctes: dict[str, ast.CTE]) -> None:
- """Visit and optimize CTEs"""
- for cte in ctes.values():
+ """Visit and optimize CTEs. A CTE can only read the ones defined before it, so the last one goes first and
+ its demands reach the earlier ones in the same pass."""
+ for cte in reversed(list(ctes.values())):
if isinstance(cte.expr, ast.SelectQuery | ast.SelectSetQuery):
self.visit(cte.expr)
@@ -306,9 +593,13 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
self._prune_columns(cte.expr)
-def pushdown_projections(node: _T_AST, context: HogQLContext) -> _T_AST:
- """Prune unused columns from asterisk expansions in subqueries"""
+def pushdown_projections(node: _T_AST, context: HogQLContext, *, narrow_properties: bool = False) -> _T_AST:
+ """Prune unused columns from asterisk expansions in subqueries. With `narrow_properties`, a subquery that exports
+ the events `properties` blob exports only the keys its readers read (see `exported_properties`)."""
if not isinstance(node, (ast.SelectQuery, ast.SelectSetQuery)):
return node
- optimizer = ProjectionPushdownOptimizer()
- return cast(_T_AST, optimizer.optimize(node))
+ optimizer = ProjectionPushdownOptimizer(context)
+ result = optimizer.optimize(node)
+ if narrow_properties and optimizer.converged:
+ narrow_exported_properties(optimizer, context)
+ return cast(_T_AST, result)
diff --git a/posthog/hogql/transforms/test/test_exported_properties.py b/posthog/hogql/transforms/test/test_exported_properties.py
new file mode 100644
index 000000000000..38ef0a664b7b
--- /dev/null
+++ b/posthog/hogql/transforms/test/test_exported_properties.py
@@ -0,0 +1,404 @@
+from posthog.test.base import (
+ APIBaseTest,
+ BaseTest,
+ ClickhouseTestMixin,
+ _create_event,
+ flush_persons_and_events,
+ materialized,
+)
+
+from parameterized import parameterized
+
+from posthog.schema import HogQLQueryModifiers
+
+from posthog.hogql.constants import HogQLDialect
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.parser import parse_select
+from posthog.hogql.printer.utils import prepare_and_print_ast
+from posthog.hogql.property_access_types import RestrictedProperty
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.clickhouse.client.escape import substitute_params
+from posthog.models import PropertyDefinition
+from posthog.models.instance_setting import override_instance_config
+
+from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
+from products.event_definitions.backend.models.property_definition import PropertyType
+
+STAR_EVENTS = "(SELECT * FROM events)"
+EMAIL_MAP = "map('email', JSONExtractRaw(events.properties, 'email'))"
+
+
+def _modifiers(pruning: bool) -> HogQLQueryModifiers:
+ return HogQLQueryModifiers(optimizeProjections=pruning)
+
+
+class TestExportedProperties(BaseTest):
+ maxDiff = None
+
+ def _print(
+ self,
+ select: str,
+ *,
+ native: bool = False,
+ pruning: bool = True,
+ restricted: set[RestrictedProperty] | None = None,
+ dialect: HogQLDialect = "clickhouse",
+ ) -> str:
+ context = HogQLContext(
+ team_id=self.team.pk,
+ enable_select_queries=True,
+ modifiers=_modifiers(pruning),
+ use_new_events_schema=native,
+ )
+ if restricted is not None:
+ context.restricted_properties = restricted
+ query, _ = prepare_and_print_ast(parse_select(select), context, dialect)
+ return substitute_params(query, context.values).replace(f"team_id, {self.team.pk})", "team_id, 420)")
+
+ @parameterized.expand(
+ [
+ (
+ "one_level",
+ f"SELECT properties.email FROM {STAR_EVENTS}",
+ f"{EMAIL_MAP} AS properties",
+ "JSONExtractRaw(properties['email'])",
+ ),
+ (
+ "passed_through_two_levels",
+ f"SELECT properties.email FROM (SELECT * FROM {STAR_EVENTS})",
+ f"SELECT properties AS properties FROM (SELECT {EMAIL_MAP} AS properties",
+ "JSONExtractRaw(properties['email'])",
+ ),
+ (
+ "aliased_passthrough",
+ "SELECT p.email FROM (SELECT properties AS p FROM events)",
+ f"{EMAIL_MAP} AS p",
+ "JSONExtractRaw(p['email'])",
+ ),
+ (
+ "cte",
+ f"WITH e AS {STAR_EVENTS} SELECT properties.email FROM e",
+ f"WITH e AS (SELECT {EMAIL_MAP} AS properties",
+ "JSONExtractRaw(e.properties['email'])",
+ ),
+ (
+ "union_branches",
+ "SELECT properties.email FROM (SELECT properties FROM events UNION ALL SELECT properties FROM events)",
+ f"{EMAIL_MAP} AS properties FROM events WHERE equals(events.team_id, 420) UNION ALL SELECT {EMAIL_MAP} AS properties",
+ "JSONExtractRaw(properties['email'])",
+ ),
+ (
+ "keys_read_at_two_levels",
+ f"SELECT properties.email FROM (SELECT * FROM {STAR_EVENTS} WHERE properties.utm_source = 'g')",
+ "map('email', JSONExtractRaw(events.properties, 'email'), 'utm_source', JSONExtractRaw(events.properties, 'utm_source')) AS properties",
+ "JSONExtractRaw(properties['utm_source'])",
+ ),
+ (
+ "nested_key",
+ "SELECT properties.a.b FROM (SELECT properties FROM events)",
+ "map('a', JSONExtractRaw(events.properties, 'a')) AS properties",
+ "JSONExtractRaw(properties['a'], 'b')",
+ ),
+ (
+ "left_join_on_the_subquery",
+ f"SELECT e.properties.email, s.n FROM {STAR_EVENTS} AS e LEFT JOIN (SELECT event AS ev, count() AS n FROM events GROUP BY ev) AS s ON s.ev = e.event",
+ f"{EMAIL_MAP} AS properties",
+ "JSONExtractRaw(e.properties['email'])",
+ ),
+ (
+ "read_inside_a_lambda",
+ "SELECT arrayMap(x -> properties.email, [1]) FROM (SELECT properties FROM events)",
+ f"{EMAIL_MAP} AS properties",
+ "x -> replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties['email'])",
+ ),
+ (
+ "unread_blob",
+ "SELECT count() FROM (SELECT properties FROM events)",
+ "accurateCast(map(), 'Map(String, Nullable(String))') AS properties",
+ "count()",
+ ),
+ (
+ "union_branches_matched_by_position",
+ "SELECT a.email FROM (SELECT properties AS a, properties AS b FROM events UNION ALL SELECT properties AS b, properties AS a FROM events)",
+ f"{EMAIL_MAP} AS b",
+ "JSONExtractRaw(a['email'])",
+ ),
+ (
+ "eight_chained_ctes",
+ f"WITH c1 AS {STAR_EVENTS}, "
+ + ", ".join(f"c{n} AS (SELECT * FROM c{n - 1})" for n in range(2, 9))
+ + " SELECT properties.email FROM c8",
+ f"WITH c1 AS (SELECT {EMAIL_MAP} AS properties",
+ "JSONExtractRaw(c8.properties['email'])",
+ ),
+ ]
+ )
+ def test_exports_only_the_keys_read(self, _name: str, select: str, exported: str, read: str) -> None:
+ printed = self._print(select)
+ assert exported in printed, printed
+ assert read in printed, printed
+ assert "events.properties AS properties" not in printed, printed
+
+ @parameterized.expand(
+ [
+ ("blob_and_key", "SELECT properties, properties.email FROM (SELECT properties FROM events)"),
+ ("json_function_on_the_blob", "SELECT JSONHas(properties, 'x') FROM (SELECT properties FROM events)"),
+ ("distinct_inner", "SELECT properties.email FROM (SELECT DISTINCT properties FROM events)"),
+ (
+ "grouped_inner",
+ "SELECT properties.email FROM (SELECT properties, count() AS c FROM events GROUP BY properties)",
+ ),
+ ("inner_order_by_alias", "SELECT p.email FROM (SELECT properties AS p FROM events ORDER BY p)"),
+ (
+ "union_distinct",
+ "SELECT properties.email FROM (SELECT properties FROM events UNION DISTINCT SELECT properties FROM events)",
+ ),
+ (
+ "intersect",
+ "SELECT properties.email FROM (SELECT properties FROM events INTERSECT SELECT properties FROM events)",
+ ),
+ (
+ "distinct_asterisk_between_the_levels",
+ "SELECT properties.email FROM (SELECT DISTINCT * FROM (SELECT properties FROM events))",
+ ),
+ (
+ "union_branch_that_cannot_narrow",
+ "SELECT properties.email FROM (SELECT properties FROM (SELECT properties FROM events) UNION ALL SELECT DISTINCT properties FROM events)",
+ ),
+ (
+ "positional_order_by_on_a_passthrough",
+ "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) ORDER BY 1 LIMIT 1)",
+ ),
+ (
+ "positional_order_by_at_the_events_level",
+ "SELECT p.email FROM (SELECT properties AS p, event FROM events ORDER BY 1 LIMIT 1)",
+ ),
+ (
+ "grouping_sets_ordinal_on_a_passthrough",
+ "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) GROUP BY GROUPING SETS ((1)))",
+ ),
+ (
+ "group_by_all_on_a_passthrough",
+ "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) GROUP BY ALL)",
+ ),
+ (
+ "second_alias_of_the_column_feeds_a_blob_branch",
+ "SELECT x.a.email, x.e FROM (SELECT a, b.email AS e FROM (SELECT properties AS a, properties AS b FROM (SELECT properties FROM events)) "
+ "UNION ALL SELECT DISTINCT properties AS a, '' AS e FROM events) AS x",
+ ),
+ ("query_output", "SELECT properties FROM (SELECT properties FROM events)"),
+ ("read_by_position", f"SELECT properties.arr[1] FROM {STAR_EVENTS}"),
+ ("native_flag_map", "SELECT properties.$feature_flags FROM (SELECT properties FROM events)", True),
+ ]
+ )
+ def test_keeps_the_blob_when_a_reader_needs_it(self, _name: str, select: str, native: bool = False) -> None:
+ printed = self._print(select, native=native)
+ assert "map(" not in printed, printed
+
+ def test_saved_view_exports_only_the_keys_read(self) -> None:
+ DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="pageviews",
+ query={"query": "SELECT event, properties FROM events WHERE event = '$pageview'"},
+ columns={
+ "event": {"clickhouse": "String", "hogql": "StringDatabaseField"},
+ "properties": {"clickhouse": "String", "hogql": "StringJSONDatabaseField"},
+ },
+ )
+ printed = self._print("SELECT properties.email FROM pageviews")
+ assert f"{EMAIL_MAP} AS properties" in printed, printed
+ assert "JSONExtractRaw(pageviews.properties['email'])" in printed, printed
+
+ @parameterized.expand([("legacy", False), ("native", True)])
+ def test_global_kill_switch_overrides_query_modifier(self, _name: str, native: bool) -> None:
+ select = f"SELECT properties.email FROM {STAR_EVENTS}"
+ unoptimized = self._print(select, native=native, pruning=False)
+ with override_instance_config("HOGQL_PROJECTION_OPTIMIZATION_ENABLED", False):
+ assert self._print(select, native=native) == unoptimized
+ with override_instance_config("HOGQL_PROJECTION_OPTIMIZATION_ENABLED", True):
+ optimized = self._print(select, native=native)
+ assert "map('email', " in optimized, optimized
+ assert "events.distinct_id AS distinct_id" not in optimized, optimized
+ assert self._print(select, native=native, pruning=False) == unoptimized
+
+ def test_native_export_reads_one_subcolumn(self) -> None:
+ printed = self._print(f"SELECT properties.utm_source FROM {STAR_EVENTS}", native=True)
+ assert "map('utm_source', " in printed, printed
+ assert "toJSONString(events.properties.utm_source)" in printed, printed
+ assert "JSONExtractKeysAndValuesRaw" not in printed, printed
+
+ @parameterized.expand(
+ [
+ ("legacy_key", False, "email", "properties.email", True),
+ ("legacy_nested", False, "obj.secret", "properties.obj.visible", True),
+ ("native_key", True, "email", "properties.email", False),
+ ("native_nested", True, "obj.secret", "properties.obj.visible", False),
+ ]
+ )
+ def test_restricted_path_reads_the_masked_blob(
+ self, _name: str, native: bool, restricted: str, read: str, narrowed: bool
+ ) -> None:
+ printed = self._print(
+ f"SELECT {read}, properties.utm_source FROM {STAR_EVENTS}",
+ native=native,
+ restricted={RestrictedProperty(name=restricted, property_type=PropertyDefinition.Type.EVENT)},
+ )
+ first_key = read.split(".")[1]
+ assert ("map(" in printed) is narrowed, printed
+ assert "JSONDropKeysPool(" in printed, printed
+ assert f"JSONExtractRaw(events.properties, '{first_key}')" not in printed, printed
+
+ def test_string_parser_still_gets_the_text(self) -> None:
+ PropertyDefinition.objects.create(
+ team=self.team, name="amount", property_type=PropertyType.Numeric, type=PropertyDefinition.Type.EVENT
+ )
+ printed = self._print(f"SELECT sum(toFloatOrZero(properties.amount)) FROM {STAR_EVENTS}")
+ assert "toFloat64OrZero(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties['amount'])" in printed, printed
+ assert "'Float64'" not in printed, printed
+
+ def test_hogql_dialect_keeps_the_blob(self) -> None:
+ printed = self._print(f"SELECT properties.email FROM {STAR_EVENTS}", dialect="hogql")
+ assert "map(" not in printed, printed
+
+
+class TestExportedPropertiesExecution(ClickhouseTestMixin, APIBaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ PropertyDefinition.objects.create(
+ team=self.team, name="amount", property_type=PropertyType.Numeric, type=PropertyDefinition.Type.EVENT
+ )
+ rows = [
+ ("2024-01-02T10:00:00", "$pageview", {"utm_source": "google", "amount": 2, "email": "[email protected]"}),
+ ("2024-01-02T11:00:00", "$pageview", {"utm_source": "", "amount": "7.5", "email": None}),
+ ("2024-01-03T10:00:00", "$pageview", {"amount": None, "nested": {"a": {"b": "deep"}}}),
+ ("2024-01-03T11:00:00", "signup", {"utm_source": "bing", "n": 1.50, "t": True}),
+ ("2024-01-03T12:00:00", "signup", {"utm_source": 'say "hi"\\now', "email": "[email protected]"}),
+ (
+ "2024-01-03T13:00:00",
+ "signup",
+ {"$feature/test": "true", "$feature_flags": {"test": "true"}, "$active_feature_flags": ["test"]},
+ ),
+ ("2023-12-31T23:59:59", "$pageview", {"utm_source": "google", "amount": 100, "a.b": "dotted"}),
+ ("2024-01-04T10:00:00", "$pageview", {}),
+ ]
+ for index, (timestamp, event, properties) in enumerate(rows):
+ _create_event(
+ team=self.team, event=event, distinct_id=f"u{index}", timestamp=timestamp, properties=properties
+ )
+ flush_persons_and_events()
+
+ def _assert_same_rows(self, select: str) -> None:
+ narrowed = execute_hogql_query(select, team=self.team, modifiers=_modifiers(True)).results
+ plain = execute_hogql_query(select, team=self.team, modifiers=_modifiers(False)).results
+ assert narrowed == plain, f"narrowed={narrowed} plain={plain}"
+ assert narrowed, "the query must match some rows for the comparison to mean anything"
+
+ @parameterized.expand(
+ [
+ (
+ "keys_and_filter",
+ f"SELECT properties.email, event FROM {STAR_EVENTS} WHERE properties.utm_source = 'google' ORDER BY 1, 2",
+ ),
+ (
+ "counts_and_key_reads",
+ f"SELECT count(), countIf(coalesce(properties.utm_source, '') != ''), uniq(person_id) FROM {STAR_EVENTS} "
+ "WHERE event = '$pageview' AND timestamp >= '2024-01-01'",
+ ),
+ (
+ "grouped_key_read",
+ f"SELECT properties.utm_source AS src, count() FROM {STAR_EVENTS} WHERE timestamp >= '2024-01-01' GROUP BY src ORDER BY src",
+ ),
+ (
+ "string_parser_on_numeric_property",
+ f"SELECT sum(toFloatOrZero(properties.amount)), arraySort(groupArray(properties.amount)) FROM {STAR_EVENTS} WHERE event = '$pageview'",
+ ),
+ (
+ "escaped_string_value",
+ f"SELECT properties.utm_source FROM {STAR_EVENTS} WHERE event = 'signup' ORDER BY 1",
+ ),
+ (
+ "empty_null_number_and_bool_values",
+ f"SELECT properties.utm_source, properties.email, properties.n, properties.t FROM {STAR_EVENTS} ORDER BY 1, 2, 3, 4",
+ ),
+ (
+ "nested_and_dotted_keys",
+ f"SELECT properties.nested.a.b, properties.`a.b` FROM {STAR_EVENTS} WHERE event = '$pageview' ORDER BY 1, 2",
+ ),
+ (
+ "passed_through_two_levels",
+ f"SELECT properties.email FROM (SELECT * FROM {STAR_EVENTS} WHERE properties.utm_source = 'google') ORDER BY 1",
+ ),
+ (
+ "aliased_passthrough",
+ "SELECT p.email FROM (SELECT properties AS p FROM events) ORDER BY 1",
+ ),
+ (
+ "cte",
+ f"WITH e AS {STAR_EVENTS} SELECT properties.email FROM e ORDER BY 1",
+ ),
+ (
+ "union_branches",
+ "SELECT properties.email FROM (SELECT properties FROM events UNION ALL SELECT properties FROM events) ORDER BY 1",
+ ),
+ (
+ "left_join_on_the_subquery",
+ f"SELECT e.properties.email, s.n FROM {STAR_EVENTS} AS e LEFT JOIN (SELECT event AS ev, count() AS n FROM events GROUP BY ev) AS s ON s.ev = e.event ORDER BY 1, 2",
+ ),
+ (
+ "unread_blob",
+ "SELECT count() FROM (SELECT properties FROM events)",
+ ),
+ (
+ "rebuilt_flag_reads",
+ f"SELECT properties.`$feature/test`, properties.$active_feature_flags FROM {STAR_EVENTS} WHERE event = 'signup' ORDER BY 1, 2",
+ ),
+ (
+ "flag_map_read",
+ f"SELECT count() FROM {STAR_EVENTS} WHERE properties.$feature_flags IS NULL",
+ ),
+ (
+ "enclosing_query_over_the_narrowed_one",
+ f"SELECT src, count() FROM (SELECT properties.utm_source AS src FROM {STAR_EVENTS} WHERE event = '$pageview') "
+ "WHERE src != '' GROUP BY src ORDER BY src",
+ ),
+ (
+ "filled_rows_read_as_missing",
+ "SELECT d, event, properties.email FROM (SELECT toDate(timestamp) AS d, event, properties FROM events ORDER BY d WITH FILL) "
+ "WHERE event = '$pageview' ORDER BY d, event, 3",
+ ),
+ ]
+ )
+ def test_narrowing_returns_the_same_rows(self, _name: str, select: str) -> None:
+ self._assert_same_rows(select)
+
+ @parameterized.expand(
+ [
+ (
+ "string_empty_value",
+ "utm_source",
+ False,
+ None,
+ f"SELECT count() FROM {STAR_EVENTS} WHERE properties.utm_source = ''",
+ ),
+ (
+ "nullable_string_escapes",
+ "utm_source",
+ True,
+ None,
+ f"SELECT properties.utm_source FROM {STAR_EVENTS} WHERE event = 'signup' ORDER BY 1",
+ ),
+ (
+ "float_under_a_string_parser",
+ "amount",
+ False,
+ "Float64",
+ f"SELECT sum(toFloatOrZero(properties.amount)), arraySort(groupArray(properties.amount)) FROM {STAR_EVENTS} WHERE event = '$pageview'",
+ ),
+ ]
+ )
+ def test_narrowing_ignores_a_materialized_column(
+ self, _name: str, property_name: str, is_nullable: bool, column_type: str | None, select: str
+ ) -> None:
+ with materialized("events", property_name, is_nullable=is_nullable, column_type=column_type):
+ self._assert_same_rows(select)
diff --git a/posthog/hogql/transforms/test/test_projection_pushdown.py b/posthog/hogql/transforms/test/test_projection_pushdown.py
index 758300578433..6c67c63496f5 100644
--- a/posthog/hogql/transforms/test/test_projection_pushdown.py
+++ b/posthog/hogql/transforms/test/test_projection_pushdown.py
@@ -118,6 +118,57 @@ class TestProjectionPushdown(BaseTest):
assert optimized.to_hogql() == self.snapshot
+ @parameterized.expand(
+ [
+ ("asterisk", "SELECT count() FROM (SELECT * FROM events) AS sub", ["uuid"]),
+ ("explicit_and_asterisk", "SELECT count() FROM (SELECT *, 1 AS one FROM events) AS sub", ["one"]),
+ ("distinct_asterisk", "SELECT count() FROM (SELECT DISTINCT * FROM events) AS sub", None),
+ ]
+ )
+ def test_unread_subquery_keeps_only_what_holds_its_rows(self, _name, query, expected_columns):
+ optimized = self._optimize(query)
+ inner_query = optimized.select_from.table
+ column_names = [self._col_name(col) for col in inner_query.select]
+ if expected_columns is None:
+ assert len(column_names) > 1
+ else:
+ assert column_names == expected_columns
+
+ @parameterized.expand(
+ [
+ ("distinct", "SELECT event FROM (SELECT DISTINCT * FROM events) AS sub"),
+ ("union_distinct", "SELECT event FROM (SELECT * FROM events UNION DISTINCT SELECT * FROM events) AS sub"),
+ ("intersect", "SELECT event FROM (SELECT * FROM events INTERSECT SELECT * FROM events) AS sub"),
+ (
+ "group_by_all",
+ "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY ALL) AS sub",
+ ),
+ (
+ "positional_group_by",
+ "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY 1, 2) AS sub",
+ ),
+ (
+ "grouping_sets_ordinals",
+ "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY GROUPING SETS ((1, 2))) AS sub",
+ ),
+ ]
+ )
+ def test_row_comparison_keeps_every_column(self, _name, query):
+ optimized = self._optimize(query)
+ inner_query = optimized.select_from.table
+ if isinstance(inner_query, ast.SelectSetQuery):
+ inner_query = inner_query.initial_select_query
+ assert len(inner_query.select) > 1
+
+ def test_set_query_branches_keep_lining_up(self):
+ optimized = self._optimize(
+ "SELECT event FROM (SELECT DISTINCT * FROM events UNION ALL SELECT * FROM events) AS sub"
+ )
+ set_query = optimized.select_from.table
+ first = set_query.initial_select_query.select
+ second = set_query.subsequent_select_queries[0].select_query.select
+ assert len(first) == len(second) > 1
+
@pytest.mark.usefixtures("unittest_snapshot")
def test_prewhere_clause(self):
"""PREWHERE clause columns should be preserved"""
diff --git a/posthog/settings/dynamic_settings.py b/posthog/settings/dynamic_settings.py
index 8f1307e28d8e..ff6d25eb4069 100644
--- a/posthog/settings/dynamic_settings.py
+++ b/posthog/settings/dynamic_settings.py
@@ -20,6 +20,13 @@ CONSTANCE_CONFIG = {
"Number of weeks recording performance events will be kept before removing them (for all projects). Storing performance events for a shorter timeframe can help reduce Clickhouse disk usage.",
int,
),
+ "HOGQL_PROJECTION_OPTIMIZATION_ENABLED": (
+ get_from_env("HOGQL_PROJECTION_OPTIMIZATION_ENABLED", True, type_cast=str_to_bool),
+ "Whether HogQL prunes unused subquery columns and narrows exported event properties. "
+ "Disable to skip both optimizations for every project, even when optimizeProjections is true. "
+ "Changes reach workers within about a minute and affect newly compiled queries only.",
+ bool,
+ ),
"HOGQL_SHARED_INSIGHT_DATABASE_ENABLED": (
get_from_env("HOGQL_SHARED_INSIGHT_DATABASE_ENABLED", True, type_cast=str_to_bool),
"Whether insight query runners share one HogQL database across execution, response printing, "
@@ -422,6 +429,7 @@ SETTINGS_ALLOWING_API_OVERRIDE = (
"GROWTH_ICP_REENRICH_DAILY_CAP",
"GROWTH_RESCORE_WEBHOOK_SECRET",
"HOGQL_DEFERRED_REVENUE_VIEWS_ENABLED",
+ "HOGQL_PROJECTION_OPTIMIZATION_ENABLED",
"HOGQL_SHARED_INSIGHT_DATABASE_ENABLED",
"HOGQL_TEAM_FLAG_CACHE_TTL_SECONDS",
"RECORDINGS_PERFORMANCE_EVENTS_TTL_WEEKS",
diff --git a/products/signals/backend/test/test_signal_queries.py b/products/signals/backend/test/test_signal_queries.py
index aff7cee240aa..f00abe7aff2e 100644
--- a/products/signals/backend/test/test_signal_queries.py
+++ b/products/signals/backend/test/test_signal_queries.py
@@ -150,8 +150,10 @@ class TestFetchSourceProductsForReports(_SignalEmbeddingsTestBase):
self._emit_version(document_id="d3", report_id="rA", source_product="errors", inserted_at=self.base)
self._emit_version(document_id="d4", report_id="rB", source_product="surveys", inserted_at=self.base)
+ # +1 for the projection kill-switch instance setting the HogQL compile reads, cold-cache here
+ # but TTL-cached per worker in production.
with (
- self.assertNumQueries(FuzzyInt(0, 1)),
+ self.assertNumQueries(FuzzyInt(1, 2)),
patch("products.signals.backend.signal_metadata.execute_hogql_query", wraps=execute_hogql_query) as query,
):
result = fetch_source_products_for_reports(