Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): read exported fields in wrapped event joins (#125074)

Sandy Spicer05b3a9b9f86c
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.md
index 9259e8647927..0a586eafaf2b 100644
--- a/docs/published/handbook/engineering/databases/hogql-python.md
+++ b/docs/published/handbook/engineering/databases/hogql-python.md
@@ -187,6 +187,10 @@ Constants starting with `[` or `{` keep the full property read because arrays an
Single-property reads serialize sub-objects with ClickHouse's JSON formatter; whole-document reads still remove declared defaults.
Grouping keys and expressions evaluated after grouping use the general property reader so conversions reference the same expression as GROUP BY.
This includes window inputs, HAVING, ORDER BY and LIMIT BY; ordinary aggregate inputs and row-level predicates retain direct-path optimizations.
+When a join constraint needs a lazy join of the joined events table, HogQL wraps that table in a subquery and the outer query reads the subquery's columns.
+The subquery exports the columns, lazy-join fields and property keys the outer query reads, found by their resolved table binding rather than by alias text.
+A property key is exported as its stored value and the outer read keeps its own conversion, so a Numeric definition and an explicit string parser read the same export.
+The whole `properties` document is exported only where the outer query reads it whole, and each wrapped table decides this on its own.
Raw SQL property readers preserve backslashes before forward slashes whether ClickHouse's `output_format_json_escape_forward_slashes` setting is enabled or disabled.
Native queries disable forward-slash escaping so object and array text retains `/`.
diff --git a/posthog/hogql/test/__snapshots__/test_query.ambr b/posthog/hogql/test/__snapshots__/test_query.ambr
index f01245a66575..aaa0a6e00ac3 100644
--- a/posthog/hogql/test/__snapshots__/test_query.ambr
+++ b/posthog/hogql/test/__snapshots__/test_query.ambr
@@ -722,7 +722,7 @@
GROUP BY person_distinct_id_overrides.distinct_id
HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0)
SETTINGS optimize_aggregation_in_order=1) AS e1__override ON equals(e1.distinct_id, e1__override.distinct_id) JOIN (
- SELECT e2.distinct_id AS distinct_id, e2.event AS event, e2.person_id AS event_person_id, if(not(empty(e2__override.distinct_id)), e2__override.person_id, e2.person_id) AS person_id
+ SELECT e2__override.distinct_id AS override___distinct_id, e2__override.person_id AS override___person_id, e2.person_id AS event_person_id, e2.event AS event
FROM events AS e2 LEFT OUTER JOIN (
SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id
FROM person_distinct_id_overrides
@@ -730,7 +730,7 @@
GROUP BY person_distinct_id_overrides.distinct_id
HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0)
SETTINGS optimize_aggregation_in_order=1) AS e2__override ON equals(e2.distinct_id, e2__override.distinct_id)
- WHERE equals(e2.team_id, 99999)) AS e2 ON equals(if(not(empty(e1__override.distinct_id)), e1__override.person_id, e1.person_id), e2.person_id)
+ WHERE equals(e2.team_id, 99999)) AS e2 ON equals(if(not(empty(e1__override.distinct_id)), e1__override.person_id, e1.person_id), if(not(empty(e2.override___distinct_id)), e2.override___person_id, e2.event_person_id))
WHERE and(equals(e1.team_id, 99999), equals(e1.event, %(hogql_val_0)s), equals(e2.event, %(hogql_val_1)s))
LIMIT 100
SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0
@@ -832,7 +832,7 @@
GROUP BY person.id
HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_0)s), person.version), plus(now64(6, %(hogql_val_1)s), toIntervalDay(1))))
SETTINGS optimize_aggregation_in_order=1) AS persons JOIN (
- SELECT events.distinct_id AS distinct_id, events.event AS event, events.person_id AS event_person_id, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id
+ SELECT events__override.distinct_id AS override___distinct_id, events__override.person_id AS override___person_id, events.person_id AS event_person_id, events.event AS event
FROM events LEFT OUTER JOIN (
SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id
FROM person_distinct_id_overrides
@@ -840,7 +840,7 @@
GROUP BY person_distinct_id_overrides.distinct_id
HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0)
SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id)
- WHERE equals(events.team_id, 99999)) AS events ON equals(persons.id, events.person_id)
+ WHERE equals(events.team_id, 99999)) AS events ON equals(persons.id, if(not(empty(events.override___distinct_id)), events.override___person_id, events.event_person_id))
LIMIT 10
SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0
diff --git a/posthog/hogql/test/__snapshots__/test_query.new_events_schema.ambr b/posthog/hogql/test/__snapshots__/test_query.new_events_schema.ambr
index ed63f67cc075..5ca03487e400 100644
--- a/posthog/hogql/test/__snapshots__/test_query.new_events_schema.ambr
+++ b/posthog/hogql/test/__snapshots__/test_query.new_events_schema.ambr
@@ -616,7 +616,7 @@
GROUP BY person_distinct_id_overrides.distinct_id
HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0)
SETTINGS optimize_aggregation_in_order=1) AS e1__override ON equals(e1.distinct_id, e1__override.distinct_id) JOIN (
- SELECT e2.distinct_id AS distinct_id, e2.event AS event, e2.person_id AS event_person_id, if(not(empty(e2__override.distinct_id)), e2__override.person_id, e2.person_id) AS person_id
+ SELECT e2__override.distinct_id AS override___distinct_id, e2__override.person_id AS override___person_id, e2.person_id AS event_person_id, e2.event AS event
FROM events_json AS e2 LEFT OUTER JOIN (
SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id
FROM person_distinct_id_overrides
@@ -624,7 +624,7 @@
GROUP BY person_distinct_id_overrides.distinct_id
HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0)
SETTINGS optimize_aggregation_in_order=1) AS e2__override ON equals(e2.distinct_id, e2__override.distinct_id)
- WHERE equals(e2.team_id, 99999)) AS e2 ON equals(if(not(empty(e1__override.distinct_id)), e1__override.person_id, e1.person_id), e2.person_id)
+ WHERE equals(e2.team_id, 99999)) AS e2 ON equals(if(not(empty(e1__override.distinct_id)), e1__override.person_id, e1.person_id), if(not(empty(e2.override___distinct_id)), e2.override___person_id, e2.event_person_id))
WHERE and(equals(e1.team_id, 99999), equals(e1.event, %(hogql_val_0)s), equals(e2.event, %(hogql_val_1)s))
LIMIT 100
SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0, json_type_escape_dots_in_keys=1, output_format_json_escape_forward_slashes=0
@@ -671,7 +671,7 @@
GROUP BY person.id
HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_0)s), person.version), plus(now64(6, %(hogql_val_1)s), toIntervalDay(1))))
SETTINGS optimize_aggregation_in_order=1) AS persons JOIN (
- SELECT events.distinct_id AS distinct_id, events.event AS event, events.person_id AS event_person_id, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id
+ SELECT events__override.distinct_id AS override___distinct_id, events__override.person_id AS override___person_id, events.person_id AS event_person_id, events.event AS event
FROM events_json AS events LEFT OUTER JOIN (
SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id
FROM person_distinct_id_overrides
@@ -679,7 +679,7 @@
GROUP BY person_distinct_id_overrides.distinct_id
HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0)
SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id)
- WHERE equals(events.team_id, 99999)) AS events ON equals(persons.id, events.person_id)
+ WHERE equals(events.team_id, 99999)) AS events ON equals(persons.id, if(not(empty(events.override___distinct_id)), events.override___person_id, events.event_person_id))
LIMIT 10
SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0, json_type_escape_dots_in_keys=1, output_format_json_escape_forward_slashes=0
diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.py
index 3a5a536ccb6a..e04b2405a596 100644
--- a/posthog/hogql/transforms/lazy_tables.py
+++ b/posthog/hogql/transforms/lazy_tables.py
@@ -9,8 +9,11 @@ from posthog.hogql.database.models import LazyJoinToAdd, LazyTableToAdd
from posthog.hogql.errors import ResolutionError
from posthog.hogql.resolver import ResolverFactory, resolve_types
from posthog.hogql.resolver_utils import get_long_table_name
+from posthog.hogql.transforms.logical_property_lowering import lower_property_access
from posthog.hogql.transforms.property_types import PropertySwapper
-from posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_expr
+from posthog.hogql.visitor import TraversingVisitor, clone_expr
+
+from posthog.dataclasses import frozen
# This mutates the nodes
@@ -63,22 +66,6 @@ def find_field_chains(node: ast.AST | None) -> list[list[str | int]]:
return finder.field_chains
-def references_table(node: ast.AST | None, table_alias: str) -> bool:
- return any(chain and chain[0] == table_alias for chain in find_field_chains(node))
-
-
-def collect_fields_from_table(node: ast.AST, table_alias: str) -> set[str]:
- return {
- chain[1]
- for chain in find_field_chains(node)
- if len(chain) >= 2 and chain[0] == table_alias and isinstance(chain[1], str)
- }
-
-
-def collect_bare_fields(node: ast.AST) -> set[str]:
- return {chain[0] for chain in find_field_chains(node) if len(chain) == 1 and isinstance(chain[0], str)}
-
-
def get_table_alias_for_lazy_join(lazy_join_type: ast.LazyJoinType) -> str | None:
table_type = lazy_join_type.table_type
while table_type:
@@ -94,60 +81,114 @@ def get_table_alias_for_lazy_join(lazy_join_type: ast.LazyJoinType) -> str | Non
return None
-def get_lazy_join_type_from_field(node: ast.Field) -> ast.LazyJoinType | None:
- if not isinstance(node.type, ast.FieldType):
- return None
- table_type = node.type.table_type
- while isinstance(table_type, ast.VirtualTableType):
- table_type = table_type.table_type
- return table_type if isinstance(table_type, ast.LazyJoinType) else None
-
-
-class LazyJoinTypeCollector(TraversingVisitor):
- def __init__(self, lazy_join_names: set[str], table_alias: str) -> None:
+def _hops_to_table(
+ table_type: ast.Type, table: ast.TableOrSelectType
+) -> list[ast.LazyJoinType | ast.VirtualTableType] | None:
+ """The lazy-join and virtual-table hops from `table` down to `table_type`, outermost first.
+
+ None when `table_type` is not bound to `table`. The same alias in a nested query is a different binding object,
+ so identity keeps the two apart where a name comparison would not.
+ """
+ hops: list[ast.LazyJoinType | ast.VirtualTableType] = []
+ current: ast.Type = table_type
+ while current is not table:
+ if not isinstance(current, (ast.LazyJoinType, ast.VirtualTableType)):
+ return None
+ hops.append(current)
+ current = current.table_type
+ hops.reverse()
+ return hops
+
+
+@frozen
+class WrappedTableExport:
+ """One column of the subquery that replaces a joined table."""
+
+ name: str
+ # The field the subquery selects from the original table, without the table alias: a column, or the document a
+ # property is read from, through any virtual-table and lazy-join hops.
+ field_chain: tuple[str, ...]
+ # The key path read from that document. It is applied after resolution so an integer array index stays an integer.
+ property_path: tuple[str | int, ...]
+ # Lower the read to an unconverted `PropertyAccess`, so the outer read keeps its own conversion context.
+ raw_property: bool
+
+
+class WrappedTableReferenceCollector(TraversingVisitor):
+ """Collects the outer reads of one table binding before a subquery replaces that table.
+
+ Reads are matched by their resolved binding rather than by the alias they were written with. That includes the
+ unqualified reads an expression field expands to, and reads through the table's virtual tables and lazy joins,
+ and it excludes a nested query's alias of the same name.
+ """
+
+ def __init__(self, table: ast.TableOrSelectType) -> None:
super().__init__()
- self.lazy_join_names = lazy_join_names
- self.table_alias = table_alias
- self.lazy_join_type_ids: set[int] = set()
-
- def visit_field(self, node: ast.Field):
- if node.chain and node.chain[0] in self.lazy_join_names:
- lazy_join_type = get_lazy_join_type_from_field(node)
- if lazy_join_type and get_table_alias_for_lazy_join(lazy_join_type) == self.table_alias:
- self.lazy_join_type_ids.add(id(lazy_join_type))
-
-
-class LazyJoinFieldFinder(TraversingVisitor):
- def __init__(self, lazy_join_type_ids: set[int]) -> None:
- super().__init__()
- self.lazy_join_type_ids = lazy_join_type_ids
- self.found: list[ast.Field] = []
-
- def visit_field(self, node: ast.Field):
- lazy_join_type = get_lazy_join_type_from_field(node)
- if lazy_join_type and id(lazy_join_type) in self.lazy_join_type_ids:
- self.found.append(node)
-
-
-class LazyJoinExpressionReplacer(CloningVisitor):
- def __init__(self, table_alias: str, subquery_type: ast.SelectQueryAliasType, lazy_join_type_ids: set[int]) -> None:
- super().__init__(clear_types=False, clear_locations=False)
- self.table_alias = table_alias
- self.subquery_type = subquery_type
- self.lazy_join_type_ids = lazy_join_type_ids
-
- def visit_call(self, node: ast.Call):
- if node.name == "if" and len(node.args) >= 2:
- finder = LazyJoinFieldFinder(self.lazy_join_type_ids)
- finder.visit(node.args[1])
- for field in finder.found:
- assert len(field.chain) == 2, f"Expected lazy join field chain of length 2, got {field.chain}"
- field_name = str(field.chain[1])
- return ast.Field(
- chain=[self.table_alias, field_name],
- type=ast.FieldType(name=field_name, table_type=self.subquery_type),
- )
- return super().visit_call(node)
+ self.table = table
+ self.exports: dict[str, WrappedTableExport] = {}
+ self.fields: dict[int, tuple[ast.FieldType, str]] = {}
+ self.properties: dict[int, tuple[ast.PropertyType, str, bool]] = {}
+ self.reads_lazy_join = False
+ self._names: dict[tuple[object, ...], str] = {}
+
+ def visit_field(self, node: ast.Field) -> None:
+ field_type = node.type
+ while isinstance(field_type, ast.FieldAliasType):
+ field_type = field_type.type
+ if isinstance(field_type, ast.PropertyType):
+ self._collect_property(field_type)
+ elif isinstance(field_type, ast.FieldType):
+ self._collect_field(field_type)
+
+ def _collect_field(self, field_type: ast.FieldType) -> None:
+ hops = _hops_to_table(field_type.table_type, self.table)
+ if hops is None:
+ return
+ field_chain = (*(hop.field for hop in hops), field_type.name)
+ name = self._export(("field", field_chain), field_chain, (), raw_property=False)
+ self.fields[id(field_type)] = (field_type, name)
+ self._note_hops(hops)
+
+ def _collect_property(self, property_type: ast.PropertyType) -> None:
+ document = property_type.field_type
+ hops = _hops_to_table(document.table_type, self.table)
+ if hops is None:
+ return
+ through_lazy_join = any(isinstance(hop, ast.LazyJoinType) for hop in hops)
+ field_chain = (*(hop.field for hop in hops), document.name)
+ property_path = tuple(property_type.chain)
+ # An integer array index and a string key with the same text are different paths.
+ key = ("property", field_chain, *((isinstance(link, int), link) for link in property_path))
+ name = self._export(key, field_chain, property_path, raw_property=not through_lazy_join)
+ self.properties[id(property_type)] = (property_type, name, through_lazy_join)
+ self._note_hops(hops)
+
+ def _note_hops(self, hops: list[ast.LazyJoinType | ast.VirtualTableType]) -> None:
+ if any(isinstance(hop, ast.LazyJoinType) for hop in hops):
+ self.reads_lazy_join = True
+
+ def _export(
+ self,
+ key: tuple[object, ...],
+ field_chain: tuple[str, ...],
+ property_path: tuple[str | int, ...],
+ raw_property: bool,
+ ) -> str:
+ name = self._names.get(key)
+ if name is None:
+ # Joining the path with the separator is not injective (`a.b` and `a___b` collide), so a taken name
+ # gets a counter. Traversal order is fixed, which keeps the generated SQL stable.
+ base = "___".join(str(link) for link in (*field_chain, *property_path))
+ name = base
+ counter = 1
+ while name in self.exports:
+ counter += 1
+ name = f"{base}_{counter}"
+ self._names[key] = name
+ self.exports[name] = WrappedTableExport(
+ name=name, field_chain=field_chain, property_path=property_path, raw_property=raw_property
+ )
+ return name
class LazyFinder(TraversingVisitor):
@@ -190,88 +231,108 @@ class LazyTableResolver(TraversingVisitor):
self.resolver_factory = resolver_factory
def _find_tables_needing_lazy_preresolution(
- self,
- node: ast.SelectQuery,
- joins_to_add: dict[str, LazyJoinToAdd],
- ) -> set[str]:
- tables_to_wrap: set[str] = set()
-
+ self, node: ast.SelectQuery, select_type: ast.SelectQueryType
+ ) -> list[str]:
+ """Aliases of joined tables whose own ON constraint reads one of their lazy joins.
+
+ A lazy join is added after the table it hangs off, so its columns are not yet in scope in that table's ON
+ constraint. The table is wrapped in a subquery that performs the lazy join first.
+ """
+ tables_to_wrap: list[str] = []
join_ptr = node.select_from
while join_ptr:
- if join_ptr.constraint is not None and isinstance(join_ptr.table, ast.Field):
- table_alias = join_ptr.alias or str(join_ptr.table.chain[0])
-
- for join_to_add in joins_to_add.values():
- if join_to_add.from_table == table_alias:
- lazy_field_name = join_to_add.lazy_join_type.field if join_to_add.lazy_join_type else None
- if lazy_field_name and references_table(join_ptr.constraint, lazy_field_name):
- tables_to_wrap.add(table_alias)
- break
-
+ if join_ptr.constraint is not None and isinstance(join_ptr.table, ast.Field) and join_ptr.type is not None:
+ for table_alias, table_type in select_type.tables.items():
+ if table_type is not join_ptr.type:
+ continue
+ collector = WrappedTableReferenceCollector(table_type)
+ collector.visit(join_ptr.constraint)
+ if collector.reads_lazy_join:
+ tables_to_wrap.append(table_alias)
+ break
join_ptr = join_ptr.next_join
-
return tables_to_wrap
def _wrap_table_for_lazy_resolution(
- self,
- node: ast.SelectQuery,
- table_alias: str,
- joins_to_add: dict[str, LazyJoinToAdd],
- select_type: ast.SelectQueryType,
+ self, node: ast.SelectQuery, table_alias: str, select_type: ast.SelectQueryType
) -> None:
+ # The scope key of a schema-qualified table (`posthog__events`) is not its written name, so match the binding.
+ original_table_type = select_type.tables[table_alias]
join_ptr = node.select_from
- while join_ptr:
- current_alias = join_ptr.alias or (
- str(join_ptr.table.chain[0]) if isinstance(join_ptr.table, ast.Field) else None
- )
- if current_alias == table_alias and isinstance(join_ptr.table, ast.Field):
- break
+ while join_ptr is not None and join_ptr.type is not original_table_type:
join_ptr = join_ptr.next_join
-
assert join_ptr is not None and isinstance(join_ptr.table, ast.Field), f"Table {table_alias} not found"
- used_fields = collect_fields_from_table(node, table_alias)
- for join_to_add in joins_to_add.values():
- if join_to_add.from_table == table_alias:
- used_fields.update(str(f) for f in join_to_add.lazy_join.from_field)
- used_fields.update(join_to_add.fields_accessed.keys())
- if join_ptr.constraint:
- used_fields.update(collect_bare_fields(join_ptr.constraint))
+ # Collect while the outer reads still identify the table they belong to.
+ collector = WrappedTableReferenceCollector(original_table_type)
+ collector.visit(node)
- select_fields: list[ast.Expr] = [
- ast.Alias(alias=field, expr=ast.Field(chain=[table_alias, field])) for field in sorted(used_fields)
- ]
subquery = ast.SelectQuery(
- select=select_fields,
+ select=[
+ ast.Alias(alias=export.name, expr=ast.Field(chain=[table_alias, *export.field_chain]))
+ for export in collector.exports.values()
+ ],
select_from=ast.JoinExpr(table=ast.Field(chain=list(join_ptr.table.chain)), alias=table_alias),
)
-
- # Resolve types and lazy tables within the subquery
subquery = cast(
ast.SelectQuery,
resolve_types(subquery, self.context, self.dialect, [], resolver_factory=self.resolver_factory),
)
- resolve_lazy_tables(subquery, self.dialect, [subquery], self.context, resolver_factory=self.resolver_factory)
assert subquery.type is not None
+ columns = [cast(ast.Alias, column) for column in subquery.select]
+ for column in columns:
+ export = collector.exports[column.alias]
+ if export.property_path:
+ self._read_property_path(subquery.type, column, export.property_path)
+ resolve_lazy_tables(subquery, self.dialect, [subquery], self.context, resolver_factory=self.resolver_factory)
+ for column in columns:
+ if collector.exports[column.alias].raw_property:
+ # The property-type pass runs after this and would convert a typed property inside the subquery.
+ # The outer read must receive the stored value, because it may sit under an explicit string parser
+ # (toFloatOrZero) that rejects the converted type, so the inner read is lowered to a raw extract now.
+ column.expr = lower_property_access(column.expr, self.context)
- # Replace the table with the subquery
join_ptr.table = subquery
join_ptr.alias = table_alias
join_ptr.type = ast.SelectQueryAliasType(alias=table_alias, select_query_type=subquery.type)
select_type.tables[table_alias] = join_ptr.type
- # Collect LazyJoinType IDs for this table's lazy joins, filtered by table_alias for self-joins
- lazy_join_field_names = {
- v.lazy_join_type.field for v in joins_to_add.values() if v.from_table == table_alias and v.lazy_join_type
- }
- collector = LazyJoinTypeCollector(lazy_join_field_names, table_alias)
- collector.visit(join_ptr.constraint)
- join_ptr.constraint = cast(
- ast.JoinConstraint,
- LazyJoinExpressionReplacer(table_alias, join_ptr.type, collector.lazy_join_type_ids).visit(
- join_ptr.constraint
- ),
- )
+ for property_type, export_name, through_lazy_join in collector.properties.values():
+ if id(property_type.field_type) in collector.fields:
+ # A key read through a select alias of the document (`properties AS p, p.amount`) shares the
+ # document's field type, which is rebound below. The property keeps its own copy of the source.
+ property_type.field_type = dataclasses.replace(property_type.field_type)
+ # The field type keeps the original table so the property-type pass still finds the property's
+ # definition, while the printer reads the exported column.
+ property_type.joined_subquery = join_ptr.type
+ property_type.joined_subquery_field_name = export_name
+ if through_lazy_join:
+ # The lazy join's subquery converted the value already, as it does without this wrapper.
+ property_type.field_type.table_type = join_ptr.type
+ for field_type, export_name in collector.fields.values():
+ field_type.name = export_name
+ field_type.table_type = join_ptr.type
+
+ def _read_property_path(
+ self, subquery_type: ast.SelectQueryType, column: ast.Alias, property_path: tuple[str | int, ...]
+ ) -> None:
+ """Turn the resolved document read in `column` into a read of `property_path` from that document.
+
+ The resolver stringifies every chain link it resolves, so the path is applied through `get_child` afterwards,
+ as array access resolution does, which keeps an integer index an integer.
+ """
+ document = column.expr
+ while isinstance(document, ast.Alias):
+ document = document.expr
+ assert isinstance(document, ast.Field) and isinstance(document.type, ast.FieldType)
+ read_type: ast.FieldType | ast.PropertyType = document.type
+ for link in property_path:
+ child = read_type.get_child(link, self.context)
+ assert isinstance(child, ast.PropertyType)
+ read_type = child
+ column.expr = ast.Field(chain=[*document.chain, *property_path], type=read_type)
+ column.type = ast.FieldAliasType(alias=column.alias, type=read_type)
+ subquery_type.columns[column.alias] = column.type
def visit_property_type(self, node: ast.PropertyType):
if node.joined_subquery is not None:
@@ -560,12 +621,17 @@ class LazyTableResolver(TraversingVisitor):
# Pre-wrap tables whose JOIN constraints reference their own lazy joins.
# This must happen BEFORE we add lazy joins to avoid forward reference issues.
- tables_to_wrap = self._find_tables_needing_lazy_preresolution(node, joins_to_add)
+ tables_to_wrap = self._find_tables_needing_lazy_preresolution(node, select_type)
+ wrapped_table_types = [select_type.tables[table_alias] for table_alias in tables_to_wrap]
for table_alias in tables_to_wrap:
- self._wrap_table_for_lazy_resolution(node, table_alias, joins_to_add, select_type)
+ self._wrap_table_for_lazy_resolution(node, table_alias, select_type)
# Remove lazy joins that are now resolved inside wrapped subqueries
- joins_to_add = {k: v for k, v in joins_to_add.items() if v.from_table not in tables_to_wrap}
+ joins_to_add = {
+ k: v
+ for k, v in joins_to_add.items()
+ if not any(_hops_to_table(v.lazy_join_type, wrapped) is not None for wrapped in wrapped_table_types)
+ }
# For all the collected tables, create the subqueries, and add them to the table.
for table_name, table_to_add in tables_to_add.items():
diff --git a/posthog/hogql/transforms/test/test_lazy_tables.py b/posthog/hogql/transforms/test/test_lazy_tables.py
index 1dd2e0c15db5..3d2d274e3c92 100644
--- a/posthog/hogql/transforms/test/test_lazy_tables.py
+++ b/posthog/hogql/transforms/test/test_lazy_tables.py
@@ -1,7 +1,16 @@
+import re
+import json
from typing import Any
+from uuid import UUID
import pytest
-from posthog.test.base import BaseTest, NewEventsSchemaSnapshotExtension
+from posthog.test.base import (
+ BaseTest,
+ NewEventsSchemaSnapshotExtension,
+ _create_event,
+ create_person_id_override_by_distinct_id,
+ flush_persons_and_events,
+)
from unittest.mock import patch
from django.conf import settings
@@ -11,14 +20,24 @@ from parameterized import parameterized
from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
+from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
-from posthog.hogql.database.database import Database
+from posthog.hogql.database.database import (
+ Database,
+ _use_person_id_from_person_overrides,
+ _use_person_properties_from_events,
+)
+from posthog.hogql.database.lazy_join_tags import PERSONS
+from posthog.hogql.database.models import LazyJoin, VirtualTable
+from posthog.hogql.database.schema.traffic_type import create_is_bot_field
from posthog.hogql.parser import parse_select
from posthog.hogql.printer import prepare_and_print_ast
+from posthog.hogql.property_access_types import RestrictedProperty
from posthog.hogql.property_metadata import PropertyMetadata
+from posthog.hogql.query import execute_hogql_query
from posthog.hogql.test.utils import pretty_print_in_tests
-from posthog.models import Organization, Team
+from posthog.models import Organization, PropertyDefinition, Team
from products.data_tools.backend.models.join import DataWarehouseJoin
@@ -60,10 +79,415 @@ class TestNamespacedLazyTables(SimpleTestCase):
return prepare_and_print_ast(parse_select(query), context, "clickhouse")[0]
+WRAPPED_JOIN = "FROM events AS initial JOIN events AS joined ON initial.person_id = joined.person_id"
+# The wrapper exports these for the join's own person-ID correction, whatever the query reads.
+JOIN_KEY_EXPORTS = {"override___distinct_id", "override___person_id", "event_person_id"}
+BUILT_IN_FIELDS = (
+ "$session_id",
+ "$window_id",
+ "$session_id_uuid",
+ "$group_0",
+ "$group_1",
+ "$group_2",
+ "$group_3",
+ "$group_4",
+)
+
+
+class TestWrappedEventJoins(SimpleTestCase):
+ @parameterized.expand([("legacy", False), ("native", True)])
+ def test_reads_exported_columns(self, _name: str, native: bool) -> None:
+ printed, _ = self._compile(
+ "SELECT " + ", ".join(f"joined.{field}" for field in BUILT_IN_FIELDS) + f" {WRAPPED_JOIN}", native=native
+ )
+ outer_select = printed.split(" FROM ", 1)[0]
+ for field in BUILT_IN_FIELDS:
+ assert f"joined.`{field}` AS `{field}`" in outer_select, printed
+ assert "JOIN (SELECT" in printed
+
+ @parameterized.expand([("legacy", False), ("native", True)])
+ def test_converts_each_outer_read_of_a_raw_property_export(self, _name: str, native: bool) -> None:
+ printed, _ = self._compile(
+ "SELECT joined.properties.amount, toFloatOrZero(joined.properties.amount), "
+ "toFloatOrZero(toString(joined.properties.amount)), joined.properties.enabled, "
+ f"joined.properties.scheduled_at {WRAPPED_JOIN}",
+ native=native,
+ )
+ outer = printed.split(" FROM ", 1)[0]
+ assert "accurateCastOrNull(joined.properties___amount, %(hogql_val_" in outer
+ assert "toFloat64OrZero(joined.properties___amount)" in outer
+ assert "toFloat64OrZero(toString(accurateCastOrNull(joined.properties___amount, %(hogql_val_" in outer
+ assert "transform(toString(joined.properties___enabled), %(hogql_val_" in outer
+ assert "parseDateTime64BestEffortOrNull(joined.properties___scheduled_at, 6, %(hogql_val_" in outer
+ inner = self._wrapper_select_list(printed, "joined", native)
+ assert inner.count("AS properties___amount") == 1
+ assert "accurateCastOrNull" not in inner
+ assert "parseDateTime64BestEffortOrNull" not in inner
+ if native:
+ assert "joined.properties.amount" in inner
+ assert "JSONExtractKeysAndValuesRaw" not in inner
+ else:
+ assert "JSONExtractRaw(joined.properties, %(hogql_val_" in inner
+
+ @parameterized.expand(
+ [
+ ("key", "joined.properties.amount", "", {"properties___amount"}),
+ ("document", "joined.properties", "", {"properties"}),
+ ("document_to_string", "toString(joined.properties)", "", {"properties"}),
+ (
+ "document_and_key",
+ "joined.properties, joined.properties.amount",
+ "",
+ {"properties", "properties___amount"},
+ ),
+ ("where_only", "count()", "WHERE joined.properties.amount > 1", {"properties___amount"}),
+ ("order_by_only", "joined.uuid", "ORDER BY joined.properties.amount", {"uuid", "properties___amount"}),
+ ]
+ )
+ def test_exports_the_document_only_for_whole_document_reads(
+ self, _name: str, select: str, clause: str, expected: set[str]
+ ) -> None:
+ for native in (False, True):
+ printed, _ = self._compile(f"SELECT {select} {WRAPPED_JOIN} {clause}", native=native)
+ assert self._exports(printed, "joined", native) == expected, printed
+
+ def test_decides_exports_per_source(self) -> None:
+ printed, _ = self._compile(
+ "SELECT b.properties, c.properties.amount FROM events AS a "
+ "JOIN events AS b ON a.person_id = b.person_id JOIN events AS c ON a.person_id = c.person_id"
+ )
+ assert self._exports(printed, "b", native=False) == {"properties"}
+ assert self._exports(printed, "c", native=False) == {"properties___amount"}
+
+ @parameterized.expand([("legacy", False), ("native", True)])
+ def test_exports_expression_field_dependencies(self, _name: str, native: bool) -> None:
+ printed, _ = self._compile(f"SELECT joined.$virt_is_bot {WRAPPED_JOIN}", native=native)
+ assert {"properties___$raw_user_agent", "properties___$ip"} <= self._exports(printed, "joined", native)
+ assert "joined.properties," not in printed.split(" FROM ", 1)[0]
+
+ def test_converts_a_key_read_through_a_document_alias(self) -> None:
+ printed, _ = self._compile(f"SELECT joined.properties AS p, p.amount + 1 {WRAPPED_JOIN}")
+ assert printed.startswith(
+ "SELECT joined.properties AS p, plus(accurateCastOrNull(joined.properties___amount, %(hogql_val_"
+ ), printed
+ assert self._exports(printed, "joined", native=False) == {"properties", "properties___amount"}
+
+ def test_wraps_a_schema_qualified_table_without_an_alias(self) -> None:
+ printed, _ = self._compile(
+ "SELECT initial.event, posthog.events.properties.amount FROM events AS initial "
+ "JOIN posthog.events ON initial.person_id = posthog.events.person_id"
+ )
+ assert "accurateCastOrNull(posthog__events.properties___amount, %(hogql_val_" in printed.split(" FROM ", 1)[0]
+ assert ") AS posthog__events ON equals(" in printed
+
+ def test_wraps_a_join_whose_constraint_needs_the_identity_join(self) -> None:
+ printed, _ = self._compile(f"SELECT joined.event {WRAPPED_JOIN}", person_overrides=False)
+ assert self._exports(printed, "joined", native=False) == {"pdi___person_id", "event"}
+ assert "AS joined__pdi ON equals(joined.distinct_id, joined__pdi.distinct_id)" in printed
+ assert printed.endswith(
+ "AS joined ON equals(initial__pdi.person_id, joined.pdi___person_id) WHERE equals(initial.team_id, 1) LIMIT 50000"
+ )
+
+ def test_corrects_the_person_id_inside_the_wrapper_only(self) -> None:
+ printed, _ = self._compile(f"SELECT joined.person_id {WRAPPED_JOIN}")
+ assert printed.startswith(
+ "SELECT if(not(empty(joined.override___distinct_id)), joined.override___person_id, "
+ "joined.event_person_id) AS person_id FROM"
+ )
+ assert printed.count("AS joined__override ON") == 1
+
+ @parameterized.expand([("identity_join", False), ("person_overrides", True)])
+ def test_converts_the_exported_timestamp_inside_the_wrapper(self, _name: str, person_overrides: bool) -> None:
+ printed, _ = self._compile(
+ "SELECT e.timestamp FROM events e LEFT JOIN events e2 "
+ "ON e.person_id = e2.person_id AND e2.timestamp >= e.timestamp "
+ "WHERE e.timestamp >= '2024-03-01' AND e.timestamp < '2024-04-01'",
+ person_overrides=person_overrides,
+ timezone="America/New_York",
+ modifiers=HogQLQueryModifiers(pushDownPredicates=False),
+ )
+ assert re.search(r"toTimeZone\(e2\.timestamp, %\(hogql_val_\d+\)s\) AS timestamp", printed), printed
+ assert re.search(r"greaterOrEquals\(e2\.timestamp, toTimeZone\(e\.timestamp,", printed), printed
+ assert re.search(r"greaterOrEquals\(e\.timestamp, ", printed), printed
+ assert re.search(r"less\(e\.timestamp, ", printed), printed
+
+ def test_keeps_property_paths_distinct(self) -> None:
+ printed, context = self._compile(
+ f"SELECT joined.properties.a.b, joined.properties.a___b, joined.properties.arr[1] {WRAPPED_JOIN}"
+ )
+ assert self._exports(printed, "joined", native=False) == {
+ "properties___a___b",
+ "properties___a___b_2",
+ "properties___arr___1",
+ }
+ assert 1 in context.values.values()
+ assert "1" not in context.values.values()
+
+ def test_keeps_nested_scopes_apart(self) -> None:
+ printed, _ = self._compile(
+ "SELECT (SELECT count() FROM events AS joined WHERE joined.properties.amount > 1) AS nested, "
+ f"joined.properties.amount {WRAPPED_JOIN}"
+ )
+ outer = printed.split(" FROM events AS initial", 1)[0]
+ assert "JSONExtractRaw(joined.properties, %(hogql_val_" in outer
+ assert "accurateCastOrNull(joined.properties___amount, %(hogql_val_" in outer
+
+ printed, _ = self._compile(f"SELECT x.v FROM (SELECT joined.properties.amount AS v {WRAPPED_JOIN}) AS x")
+ assert printed.startswith("SELECT x.v AS v FROM (SELECT accurateCastOrNull(joined.properties___amount, ")
+ assert "AS properties___amount" in printed
+
+ @parameterized.expand([("legacy", False), ("native", True)])
+ def test_reads_person_properties_through_the_wrapper(self, _name: str, native: bool) -> None:
+ printed, _ = self._compile(
+ f"SELECT joined.person.properties.age {WRAPPED_JOIN}", native=native, person_properties="on_events"
+ )
+ assert printed.startswith("SELECT accurateCastOrNull(joined.poe___properties___age, %(hogql_val_")
+ assert "joined.person_properties" in self._wrapper_select_list(printed, "joined", native)
+
+ printed, _ = self._compile(
+ f"SELECT joined.person.properties.age {WRAPPED_JOIN}", native=native, person_properties="joined"
+ )
+ assert printed.startswith("SELECT joined.person___properties___age AS age FROM")
+ assert printed.count("accurateCastOrNull") == 1
+ assert "AS joined__person ON" in printed
+
+ @parameterized.expand([("legacy", False), ("native", True)])
+ def test_keeps_property_restrictions_inside_the_wrapper(self, _name: str, native: bool) -> None:
+ restricted = {RestrictedProperty(name="amount", property_type=PropertyDefinition.Type.EVENT)}
+ printed, _ = self._compile(
+ f"SELECT joined.properties.amount, joined.properties.label {WRAPPED_JOIN}",
+ native=native,
+ restricted=restricted,
+ )
+ inner = self._wrapper_select_list(printed, "joined", native)
+ assert "NULL AS properties___amount" in inner
+ assert "AS properties___label" in inner and "NULL AS properties___label" not in inner
+
+ printed, _ = self._compile(f"SELECT joined.properties {WRAPPED_JOIN}", native=native, restricted=restricted)
+ assert "JSONDropKeysPool(" in self._wrapper_select_list(printed, "joined", native)
+
+ def _compile(
+ self,
+ query: str,
+ native: bool = False,
+ person_properties: str | None = None,
+ restricted: set[RestrictedProperty] | None = None,
+ person_overrides: bool = True,
+ timezone: str = "UTC",
+ modifiers: HogQLQueryModifiers | None = None,
+ ) -> tuple[str, HogQLContext]:
+ database = Database(timezone=timezone)
+ events = database.get_table("events")
+ events.fields["$virt_is_bot"] = create_is_bot_field(name="$virt_is_bot")
+ if person_overrides:
+ _use_person_id_from_person_overrides(database)
+ if person_properties == "on_events":
+ _use_person_properties_from_events(database)
+ poe = events.fields["poe"]
+ assert isinstance(poe, VirtualTable)
+ poe.fields["id"] = events.fields["person_id"]
+ elif person_properties == "joined":
+ events.fields["person"] = LazyJoin(
+ from_field=["person_id"], join_table=database.get_table("persons"), resolver=PERSONS
+ )
+ context = HogQLContext(
+ team_id=1,
+ team=Team(id=1, organization=Organization(), timezone=timezone),
+ database=database,
+ enable_select_queries=True,
+ use_new_events_schema=native,
+ apply_events_retention_floor=False,
+ restricted_properties=restricted or set(),
+ modifiers=modifiers or HogQLQueryModifiers(),
+ )
+ metadata = PropertyMetadata(
+ event_properties={
+ "amount": {"type": "Numeric"},
+ "enabled": {"type": "Boolean"},
+ "scheduled_at": {"type": "DateTime"},
+ },
+ person_properties={"age": {"type": "Numeric"}},
+ )
+ with patch("posthog.hogql.transforms.property_types.load_property_metadata", return_value=metadata):
+ printed, _ = prepare_and_print_ast(parse_select(query), context, "clickhouse")
+ return printed, context
+
+ def _wrapper_select_list(self, printed: str, alias: str, native: bool) -> str:
+ events_table = "events_json" if native else "events"
+ head = printed.split(f" FROM {events_table} AS {alias} ", 1)[0]
+ return head.rsplit("JOIN (SELECT ", 1)[1]
+
+ def _exports(self, printed: str, alias: str, native: bool) -> set[str]:
+ select_list = self._wrapper_select_list(printed, alias, native)
+ names = {name.strip("`") for name in re.findall(r" AS (`[^`]+`|\w+)", select_list)}
+ return names - JOIN_KEY_EXPORTS
+
+
class TestLazyJoins(BaseTest):
snapshot: Any
maxDiff = None
+ @parameterized.expand(
+ [
+ (
+ "values",
+ "related.$session_id, related.properties.label, related.properties.amount, "
+ "related.properties.enabled, toString(related.properties.scheduled_at)",
+ ("00000000-0000-4000-8000-000000000002", "ready", 7.0, True, "2024-01-02 03:04:05.000000"),
+ ),
+ (
+ "aggregates",
+ "sum(related.properties.amount), countIf(related.properties.enabled), "
+ "toString(max(related.properties.scheduled_at))",
+ (7.0, 1, "2024-01-02 03:04:05.000000"),
+ ),
+ (
+ "mixed_contexts_on_text",
+ "related.properties.rating, toFloatOrZero(related.properties.rating)",
+ (None, 0.0),
+ ),
+ (
+ "paths",
+ "related.properties.a.b, related.properties.a___b, related.properties.arr[1]",
+ ("nested", "flat", "first"),
+ ),
+ (
+ "window",
+ "sum(related.properties.amount) OVER (PARTITION BY related.$session_id)",
+ (7.0,),
+ ),
+ ]
+ )
+ def test_wrapped_events_join_executes_exported_property_reads(
+ self, _name: str, select: str, expected: tuple[object, ...]
+ ) -> None:
+ for name, property_type in [
+ ("amount", "Numeric"),
+ ("rating", "Numeric"),
+ ("enabled", "Boolean"),
+ ("scheduled_at", "DateTime"),
+ ]:
+ PropertyDefinition.objects.create(team=self.team, name=name, property_type=property_type)
+ event_uuid = _create_event(
+ team=self.team,
+ event="joined-event",
+ distinct_id="joined-user",
+ person_id=UUID("00000000-0000-4000-8000-000000000001"),
+ properties={
+ "$session_id": "00000000-0000-4000-8000-000000000002",
+ "label": "ready",
+ "amount": "7",
+ "rating": "seven",
+ "enabled": "true",
+ "scheduled_at": "2024-01-02T03:04:05Z",
+ "a": {"b": "nested"},
+ "a___b": "flat",
+ "arr": ["first", "second"],
+ },
+ )
+ result = execute_hogql_query(
+ f"SELECT {select} FROM events AS source "
+ "JOIN events AS related ON source.person_id = related.person_id "
+ "WHERE source.uuid = {event_uuid} AND related.uuid = {event_uuid}",
+ team=self.team,
+ placeholders={"event_uuid": ast.Constant(value=event_uuid)},
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
+ ),
+ )
+ assert [tuple(row) for row in result.results] == [expected]
+
+ def test_wrapped_events_join_converts_a_key_read_through_a_document_alias(self) -> None:
+ PropertyDefinition.objects.create(team=self.team, name="amount", property_type="Numeric")
+ event_uuid = _create_event(
+ team=self.team,
+ event="priced",
+ distinct_id="user",
+ person_id=UUID("00000000-0000-4000-8000-000000000001"),
+ properties={"amount": "7"},
+ )
+ result = execute_hogql_query(
+ "SELECT related.properties AS p, p.amount + 1 FROM events AS source "
+ "JOIN events AS related ON source.person_id = related.person_id "
+ "WHERE source.uuid = {event_uuid} AND related.uuid = {event_uuid}",
+ team=self.team,
+ placeholders={"event_uuid": ast.Constant(value=event_uuid)},
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
+ ),
+ )
+ assert [(json.loads(document)["amount"], value) for document, value in result.results] == [("7", 8.0)]
+
+ def test_wrapped_events_join_reads_each_alias_from_its_own_source(self) -> None:
+ PropertyDefinition.objects.create(team=self.team, name="amount", property_type="Numeric")
+ person_id = UUID("00000000-0000-4000-8000-000000000001")
+ first = _create_event(
+ team=self.team, event="first", distinct_id="user", person_id=person_id, properties={"amount": "7"}
+ )
+ second = _create_event(
+ team=self.team, event="second", distinct_id="user", person_id=person_id, properties={"amount": "11"}
+ )
+ result = execute_hogql_query(
+ "SELECT b.properties.amount, c.properties.amount, JSONExtractKeys(c.properties) FROM events AS a "
+ "JOIN events AS b ON a.person_id = b.person_id JOIN events AS c ON a.person_id = c.person_id "
+ "WHERE a.uuid = {first} AND b.uuid = {first} AND c.uuid = {second}",
+ team=self.team,
+ placeholders={"first": ast.Constant(value=first), "second": ast.Constant(value=second)},
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
+ ),
+ )
+ assert [tuple(row) for row in result.results] == [(7.0, 11.0, ["amount"])]
+
+ def test_wrapped_events_join_corrects_the_person_id_inside_the_wrapper(self) -> None:
+ merged_from = _create_event(
+ team=self.team,
+ event="before-merge",
+ distinct_id="merged_from",
+ person_id=UUID("00000000-0000-4000-8000-000000000001"),
+ )
+ merged_to = _create_event(
+ team=self.team,
+ event="after-merge",
+ distinct_id="merged_to",
+ person_id=UUID("00000000-0000-4000-8000-000000000002"),
+ )
+ flush_persons_and_events()
+ create_person_id_override_by_distinct_id("merged_from", "merged_to", self.team.pk)
+
+ result = execute_hogql_query(
+ "SELECT related.distinct_id, related.person_id FROM events AS source "
+ "JOIN events AS related ON source.person_id = related.person_id "
+ "WHERE source.uuid = {merged_to} AND related.uuid = {merged_from}",
+ team=self.team,
+ placeholders={"merged_from": ast.Constant(value=merged_from), "merged_to": ast.Constant(value=merged_to)},
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
+ ),
+ )
+ assert [tuple(row) for row in result.results] == [("merged_from", UUID("00000000-0000-4000-8000-000000000002"))]
+
+ def test_wrapped_events_left_join_keeps_unmatched_rows(self) -> None:
+ PropertyDefinition.objects.create(team=self.team, name="amount", property_type="Numeric")
+ event_uuid = _create_event(
+ team=self.team,
+ event="alone",
+ distinct_id="user",
+ person_id=UUID("00000000-0000-4000-8000-000000000001"),
+ properties={"amount": "7"},
+ )
+ result = execute_hogql_query(
+ "SELECT source.event, related.properties.amount, related.event FROM events AS source "
+ "LEFT JOIN events AS related ON source.person_id = related.person_id AND related.event = 'other' "
+ "WHERE source.uuid = {event_uuid}",
+ team=self.team,
+ placeholders={"event_uuid": ast.Constant(value=event_uuid)},
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
+ ),
+ )
+ assert [tuple(row) for row in result.results] == [("alone", None, "")]
+
def _schema_snapshot(self, printed: str):
self.snapshot.session.pytest_session.config.option.warn_unused_snapshots = True
if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA and "events_json" in printed:
diff --git a/posthog/hogql/transforms/test/test_property_types.py b/posthog/hogql/transforms/test/test_property_types.py
index 805847821502..cef29e066c4e 100644
--- a/posthog/hogql/transforms/test/test_property_types.py
+++ b/posthog/hogql/transforms/test/test_property_types.py
@@ -112,9 +112,10 @@ class _NewEventsSchemaArraySubcolumnsHelpers:
use_new_events_schema: bool | None = None,
restricted_properties: set[RestrictedProperty] | None = None,
property_groups_mode: PropertyGroupsMode | None = None,
+ event_properties: dict[str, dict[str, str | None]] | None = None,
) -> str:
expr = parse_select(select)
- context = self._context(use_new_events_schema)
+ context = self._context(use_new_events_schema, event_properties)
if restricted_properties is not None:
context.restricted_properties = restricted_properties
if property_groups_mode is not None:
@@ -202,6 +203,42 @@ class TestNewEventsSchemaArraySubcolumns(_NewEventsSchemaArraySubcolumnsHelpers,
assert "events.properties" not in restricted
assert "events.person_properties" not in restricted
+ @parameterized.expand(
+ [
+ ("aliased", "SELECT s.properties.amount = 'seven' FROM (SELECT properties FROM events) AS s"),
+ (
+ "nested",
+ "SELECT t.properties.amount = 'seven' FROM (SELECT * FROM (SELECT properties FROM events) AS s) AS t",
+ ),
+ ("unaliased", "SELECT properties.amount = 'seven' FROM (SELECT properties FROM events)"),
+ (
+ "union",
+ "SELECT s.properties.amount = 'seven' "
+ "FROM (SELECT properties FROM events UNION ALL SELECT properties FROM events) AS s",
+ ),
+ (
+ "cte",
+ "WITH docs AS (SELECT properties FROM events) SELECT docs.properties.amount = 'seven' FROM docs",
+ ),
+ ]
+ )
+ def test_property_read_from_a_user_document_subquery_stays_a_string(self, _name: str, select: str) -> None:
+ for use_new_events_schema in (False, True):
+ printed = self._print_select(
+ select, use_new_events_schema=use_new_events_schema, event_properties={"amount": {"type": "Numeric"}}
+ )
+ assert "accurateCastOrNull" not in printed, printed
+ assert "JSONExtractRaw(" in printed
+
+ def test_property_converted_inside_a_user_subquery_is_not_converted_again(self) -> None:
+ printed = self._print_select(
+ "SELECT s.amount = 7 FROM (SELECT properties.amount AS amount FROM events) AS s",
+ use_new_events_schema=False,
+ event_properties={"amount": {"type": "Numeric"}},
+ )
+ assert printed.count("accurateCastOrNull") == 1
+ assert "ifNull(equals(s.amount, 7), 0)" in printed
+
def test_native_string_predicate_inside_an_unaliased_column(self) -> None:
native = self._print_select("SELECT countIf(properties.value = '5') FROM events")
assert (
@@ -2675,7 +2712,7 @@ class TestTimezoneIndexPruning(ClickhouseTestMixin, BaseTest):
f"Expected PK to use {expected_timestamp_key}, got Keys={pk_keys}"
)
- def _compile_hogql(self, hogql: str, timezone: str = "UTC") -> tuple[str, dict]:
+ def _compile_hogql(self, hogql: str, timezone: str = "UTC") -> tuple[str, dict[str, object]]:
self.team.timezone = timezone
self.team.save()
# This class asserts the structural shape of toTimeZone stripping (a PropertySwapper concern).
@@ -2781,24 +2818,6 @@ class TestTimezoneIndexPruning(ClickhouseTestMixin, BaseTest):
assert re.search(r"greaterOrEquals\([\w.]*scheduled_at, ", sql), sql
assert not re.search(r"ifNull\(greaterOrEquals\([\w.]*scheduled_at", sql), sql
- def test_toTimeZone_not_stripped_in_join_on(self):
- """toTimeZone should NOT be stripped from JOIN ON comparisons — only WHERE benefits from pruning."""
- sql, _ = self._compile_hogql(
- "SELECT e.timestamp FROM events e LEFT JOIN events e2 "
- "ON e.person_id = e2.person_id AND e2.timestamp >= e.timestamp "
- "WHERE e.timestamp >= '2024-03-01' AND e.timestamp < '2024-04-01'",
- timezone="America/New_York",
- )
- # The JOIN ON greaterOrEquals should still have toTimeZone wrapping
- assert re.search(r"greaterOrEquals\(toTimeZone\(", sql), (
- f"Expected toTimeZone preserved in JOIN ON greaterOrEquals, got:\n{sql}"
- )
- # The WHERE comparisons should have toTimeZone stripped (bare e.timestamp)
- assert re.search(r"greaterOrEquals\(e\.timestamp,", sql), (
- f"Expected bare e.timestamp in WHERE greaterOrEquals, got:\n{sql}"
- )
- assert re.search(r"less\(e\.timestamp,", sql), f"Expected bare e.timestamp in WHERE less, got:\n{sql}"
-
def test_toTimeZone_not_stripped_inside_function_calls(self):
"""toTimeZone should NOT be stripped from comparisons inside function calls."""
sql, _ = self._compile_hogql(