Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): export virtual-table fields under their own lazy table names (#127797)

Phil Haack7dbff7774859
diff --git a/posthog/hogql/transforms/flag_calls_union.py b/posthog/hogql/transforms/flag_calls_union.py
index fcc3d2ca9fd8..cd52f376c113 100644
--- a/posthog/hogql/transforms/flag_calls_union.py
+++ b/posthog/hogql/transforms/flag_calls_union.py
@@ -730,12 +730,12 @@ class _UnionChainCollector(SyntaxVisitor):
if read is not None:
chain = read[1]
self.chains.append(chain)
- # Lazy table resolution names a whole-field read after the field, whichever table the field is on.
- # It names a property read by joining the path with `___`, so `a.b` and `a___b` share a name.
- name = node.type.name if isinstance(node.type, ast.FieldType) else "___".join(str(link) for link in chain)
- first = self.exports.setdefault((id(self._select), name), chain)
- if first != chain and self.export_collision is None:
- self.export_collision = (first, chain)
+ if not isinstance(node.type, ast.FieldType):
+ # Lazy table resolution names a property read by joining the path with `___`, so `a.b` and `a___b` share a name.
+ name = "___".join(str(link) for link in chain)
+ first = self.exports.setdefault((id(self._select), name), chain)
+ if first != chain and self.export_collision is None:
+ self.export_collision = (first, chain)
return
field_type = node.type.field_type if isinstance(node.type, ast.PropertyType) else node.type
if not isinstance(field_type, ast.FieldType):
diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.py
index f7ebf6d84ced..f3ed8dc84997 100644
--- a/posthog/hogql/transforms/lazy_tables.py
+++ b/posthog/hogql/transforms/lazy_tables.py
@@ -532,6 +532,23 @@ class LazyTableResolver(TraversingVisitor):
# Field types retain the original tables after the scope replaces them with subqueries.
original_select_type = ast.SelectQueryType(tables=dict(select_type.tables))
+ # Virtual-table fields take their own export name, which must not equal the name of any other export.
+ export_names: dict[int, str] = {}
+
+ def allocate_export_name(
+ accessed: dict[str, list[str | int]], field: ast.FieldType, chain: list[str | int]
+ ) -> str:
+ if not isinstance(field.table_type, ast.VirtualTableType):
+ return field.name
+ base = "___".join(str(link) for link in chain)
+ name = base
+ counter = 1
+ while name in accessed and accessed[name] != chain:
+ counter += 1
+ name = f"{base}_{counter}"
+ export_names[id(field)] = name
+ return name
+
# First properties, then fields. This way we always get the smallest units to query first.
matched_properties: list[ast.PropertyType | ast.FieldType] = [
property for property in field_collector if isinstance(property, ast.PropertyType)
@@ -539,6 +556,10 @@ class LazyTableResolver(TraversingVisitor):
matched_fields: list[ast.PropertyType | ast.FieldType] = [
field for field in field_collector if isinstance(field, ast.FieldType)
]
+ # Virtual-table fields go last, so each one picks a name that no other export has taken.
+ matched_fields.sort(
+ key=lambda field: isinstance(field, ast.FieldType) and isinstance(field.table_type, ast.VirtualTableType)
+ )
sorted_properties: list[ast.PropertyType | ast.FieldType] = matched_properties + matched_fields
# Look for tables without requested fields to support cases like `select count() from table`
@@ -646,7 +667,9 @@ class LazyTableResolver(TraversingVisitor):
property.joined_subquery_field_name = "___".join(str(x) for x in chain)
new_join.fields_accessed[property.joined_subquery_field_name] = chain
else:
- new_join.fields_accessed[field.name] = chain
+ new_join.fields_accessed[allocate_export_name(new_join.fields_accessed, field, chain)] = (
+ chain
+ )
elif isinstance(table_type, ast.LazyTableType):
table_name = get_long_table_name(select_type, table_type)
if table_name not in tables_to_add:
@@ -667,7 +690,9 @@ class LazyTableResolver(TraversingVisitor):
property.joined_subquery_field_name = "___".join(str(x) for x in lazy_chain)
new_table.fields_accessed[property.joined_subquery_field_name] = lazy_chain
else:
- new_table.fields_accessed[field.name] = lazy_chain
+ new_table.fields_accessed[
+ allocate_export_name(new_table.fields_accessed, field, lazy_chain)
+ ] = lazy_chain
elif isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):
# Widen mypy's view of the inner type: the annotated union is
# `TableType | LazyTableType`, but the LazyJoinType branch below is a
@@ -701,7 +726,9 @@ class LazyTableResolver(TraversingVisitor):
property.joined_subquery_field_name = "___".join(str(x) for x in field_chain)
new_join.fields_accessed[property.joined_subquery_field_name] = field_chain
else:
- new_join.fields_accessed[field.name] = field_chain
+ new_join.fields_accessed[
+ allocate_export_name(new_join.fields_accessed, field, field_chain)
+ ] = field_chain
elif isinstance(inner_table_type, ast.LazyTableType):
table_name = get_long_table_name(select_type, table_type)
if table_name not in tables_to_add:
@@ -723,7 +750,9 @@ class LazyTableResolver(TraversingVisitor):
property.joined_subquery_field_name = "___".join(str(x) for x in chain)
new_table.fields_accessed[property.joined_subquery_field_name] = chain
else:
- new_table.fields_accessed[field.name] = chain
+ new_table.fields_accessed[
+ allocate_export_name(new_table.fields_accessed, field, chain)
+ ] = chain
# Make sure we also add fields we will use for the join's "ON" condition into the list of fields accessed.
# Without this "pdi.person.id" won't work if you did not ALSO select "pdi.person_id" explicitly for the join.
@@ -952,6 +981,10 @@ class LazyTableResolver(TraversingVisitor):
continue
if isinstance(field_or_property, ast.FieldType):
+ # A field still on its virtual table reads the export name allocated above.
+ # A field that a wrapper subquery already rebound keeps the name the wrapper gave it.
+ if isinstance(field_or_property.table_type, ast.VirtualTableType):
+ field_or_property.name = export_names.get(id(field_or_property), field_or_property.name)
field_or_property.table_type = table_type
elif isinstance(field_or_property, ast.PropertyType):
field_or_property.field_type.table_type = table_type
diff --git a/posthog/hogql/transforms/test/test_flag_calls_union.py b/posthog/hogql/transforms/test/test_flag_calls_union.py
index 4a2d55c649d9..615da6e38e91 100644
--- a/posthog/hogql/transforms/test/test_flag_calls_union.py
+++ b/posthog/hogql/transforms/test/test_flag_calls_union.py
@@ -1,4 +1,5 @@
import re
+import json
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any, cast
@@ -657,15 +658,14 @@ class TestReadFlagCallsWithUnion(BaseTest):
False,
mode=ONLY,
),
- # Each scan exports one created_at, so the two fields never share a column.
+ # Each scan exports its own property columns, so paths that join to one name in separate scans never share a column.
param(
- "fields_of_one_name_in_separate_scans",
- f"SELECT created_at FROM events WHERE {MIXED} UNION ALL "
- f"SELECT person.created_at FROM events WHERE {MIXED}",
+ "property_paths_of_one_name_in_separate_scans",
+ f"SELECT properties.a.b FROM events WHERE {MIXED} UNION ALL "
+ f"SELECT properties.a___b FROM events WHERE {MIXED}",
True,
True,
mode=ONLY,
- persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,
),
param(
"alias_of_event_in_a_mixed_filter_on_mode_2",
@@ -778,21 +778,11 @@ class TestReadFlagCallsWithUnion(BaseTest):
"probe_join",
warehouse_join=True,
),
- (
- "field_and_virtual_field_of_one_name",
- f"SELECT created_at, person.created_at FROM events WHERE {MIXED}",
- "created_at and poe.created_at",
- ),
(
"properties_whose_paths_join_to_one_name",
f"SELECT properties.a.b, properties.a___b FROM events WHERE {MIXED}",
"properties.a.b and properties.a___b",
),
- (
- "properties_and_virtual_properties",
- f"SELECT properties, person.properties FROM events WHERE {MIXED}",
- "properties and poe.properties",
- ),
param(
"property_in_a_typed_materialized_column",
f"SELECT properties.probe FROM events WHERE {MIXED}",
@@ -1578,6 +1568,31 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):
assert response.results == [("$feature_flag_called", "flag_evaluations", 12.5), ("$pageview", None, 20.0)]
+ @parameterized.expand(
+ [
+ ("mode_1", FlagEvaluationsMode.READ_FLAG_EVALUATIONS),
+ ("mode_2", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY),
+ ]
+ )
+ def test_fields_of_one_name_on_events_and_person_return_their_own_values(
+ self, _name: str, mode: FlagEvaluationsMode
+ ):
+ response = self._run(
+ f"SELECT event, properties.source, person.properties.email, properties, person.properties, created_at, "
+ f"person.created_at FROM events WHERE {MIXED} AND distinct_id = 'recent' "
+ "AND timestamp >= now() - INTERVAL 5 DAY ORDER BY event",
+ mode,
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,
+ )
+
+ flag_call, pageview = response.results
+ assert flag_call[:3] == ("$feature_flag_called", "flag_evaluations", "[email protected]")
+ assert json.loads(flag_call[3])["source"] == "flag_evaluations"
+ assert json.loads(flag_call[4]) == {"email": "[email protected]"}
+ assert pageview[:3] == ("$pageview", None, "[email protected]")
+ assert json.loads(pageview[4]) == {"email": "[email protected]"}
+ assert all(row[6] is not None and row[5] != row[6] for row in response.results)
+
@parameterized.expand([("read_flag_evaluations", READ), ("flag_evaluations_only", ONLY)])
def test_json_extract_string_reads_the_raw_json_value(self, _name: str, mode: FlagEvaluationsMode):
for name, kind in (
@@ -1782,16 +1797,7 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):
} <= set(accepted)
types: dict[bool, dict[str, str]] = {False: {}, True: {}}
# One query per few fields keeps each union within the memory of a local ClickHouse.
- # Two fields of one name, such as goe_0.key and goe_1.key, would return one column, so they go in different queries.
- batches: list[list[str]] = []
- for name in accepted:
- batch = next(
- (b for b in batches if len(b) < 8 and all(n.rsplit(".", 1)[-1] != name.rsplit(".", 1)[-1] for n in b)),
- None,
- )
- if batch is None:
- batches.append(batch := [])
- batch.append(name)
+ batches = [accepted[i : i + 8] for i in range(0, len(accepted), 8)]
for names in batches:
types_query = f"SELECT {', '.join(f'toTypeName(any({name}))' for name in names)} FROM events WHERE {MIXED}"
for union in (False, True):
diff --git a/posthog/hogql/transforms/test/test_lazy_tables.py b/posthog/hogql/transforms/test/test_lazy_tables.py
index 9653dd05eeb5..45f9004feedc 100644
--- a/posthog/hogql/transforms/test/test_lazy_tables.py
+++ b/posthog/hogql/transforms/test/test_lazy_tables.py
@@ -29,7 +29,14 @@ from posthog.hogql.database.database import (
_use_person_properties_from_events,
)
from posthog.hogql.database.lazy_join_tags import PERSONS
-from posthog.hogql.database.models import LazyJoin, VirtualTable
+from posthog.hogql.database.models import (
+ IntegerDatabaseField,
+ LazyJoin,
+ LazyTable,
+ LazyTableToAdd,
+ TableNode,
+ VirtualTable,
+)
from posthog.hogql.database.schema.traffic_type import create_is_bot_field
from posthog.hogql.parser import parse_select
from posthog.hogql.printer import prepare_and_print_ast
@@ -164,6 +171,70 @@ BUILT_IN_FIELDS = (
)
+class _ProbeLazyTable(LazyTable):
+ fields: dict = {
+ "x": IntegerDatabaseField(name="x"),
+ "v___x": IntegerDatabaseField(name="v___x"),
+ "v": VirtualTable(fields={"x": IntegerDatabaseField(name="x")}),
+ }
+
+ def lazy_select(self, table_to_add: LazyTableToAdd, context, node):
+ return ast.SelectQuery(
+ select=[ast.Alias(alias=name, expr=ast.Constant(value=name)) for name in table_to_add.fields_accessed]
+ )
+
+ def to_printed_clickhouse(self, context):
+ return "probe"
+
+ def to_printed_hogql(self):
+ return "probe"
+
+
+class TestVirtualFieldExports(SimpleTestCase):
+ @parameterized.expand([("bare", "probe", "probe"), ("aliased", "probe AS p", "p")])
+ def test_virtual_field_export_never_takes_the_name_of_another_export(
+ self, _name: str, source: str, prefix: str
+ ) -> None:
+ database = Database()
+ database.tables.children["probe"] = TableNode(name="probe", table=_ProbeLazyTable())
+ context = HogQLContext(
+ team_id=1,
+ team=Team(id=1, organization=Organization()),
+ database=database,
+ enable_select_queries=True,
+ use_new_events_schema=False,
+ )
+
+ with patch("posthog.hogql.transforms.property_types.load_property_metadata", return_value=PropertyMetadata()):
+ printed, _ = prepare_and_print_ast(
+ parse_select(f"SELECT {prefix}.x, {prefix}.v.x, {prefix}.v___x FROM {source}"), context, "clickhouse"
+ )
+
+ values = re.findall(r"%\(hogql_val_\d+\)s AS (`?\w+`?)", printed)
+ assert sorted(name.strip("`") for name in values) == ["v___x", "v___x_2", "x"]
+ assert printed.startswith(f"SELECT {prefix}.x, {prefix}.v___x_2 AS x, {prefix}.v___x AS v___x FROM")
+
+ def test_virtual_field_read_through_a_wrapped_lazy_join_names_the_column_the_wrapper_exports(self) -> None:
+ context = HogQLContext(
+ team_id=1,
+ team=Team(id=1, organization=Organization()),
+ database=Database(),
+ enable_select_queries=True,
+ apply_events_retention_floor=False,
+ use_new_events_schema=False,
+ )
+ query = (
+ "SELECT r.events.poe.created_at, r.events.created_at FROM events AS e "
+ "JOIN raw_session_replay_events AS r ON r.events.event = e.event"
+ )
+
+ with patch("posthog.hogql.transforms.property_types.load_property_metadata", return_value=PropertyMetadata()):
+ printed, _ = prepare_and_print_ast(parse_select(query), context, "clickhouse")
+
+ assert printed.startswith("SELECT r.events___poe___created_at, r.events___created_at AS created_at FROM")
+ assert "AS events___poe___created_at" in printed
+
+
class TestWrappedEventJoins(SimpleTestCase):
@parameterized.expand([("legacy", False), ("native", True)])
def test_reads_exported_columns(self, _name: str, native: bool) -> None: