:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(flags): export virtual-table fields under their own lazy table names (#127797)
Phil Haack7dbff7774859
diff --git a/posthog/hogql/transforms/flag_calls_union.py b/posthog/hogql/transforms/flag_calls_union.pyindex fcc3d2ca9fd8..cd52f376c113 100644--- a/posthog/hogql/transforms/flag_calls_union.py+++ b/posthog/hogql/transforms/flag_calls_union.py@@ -730,12 +730,12 @@ class _UnionChainCollector(SyntaxVisitor):if read is not None:chain = read[1]self.chains.append(chain)- # Lazy table resolution names a whole-field read after the field, whichever table the field is on.- # It names a property read by joining the path with `___`, so `a.b` and `a___b` share a name.- name = node.type.name if isinstance(node.type, ast.FieldType) else "___".join(str(link) for link in chain)- first = self.exports.setdefault((id(self._select), name), chain)- if first != chain and self.export_collision is None:- self.export_collision = (first, chain)+ if not isinstance(node.type, ast.FieldType):+ # Lazy table resolution names a property read by joining the path with `___`, so `a.b` and `a___b` share a name.+ name = "___".join(str(link) for link in chain)+ first = self.exports.setdefault((id(self._select), name), chain)+ if first != chain and self.export_collision is None:+ self.export_collision = (first, chain)returnfield_type = node.type.field_type if isinstance(node.type, ast.PropertyType) else node.typeif not isinstance(field_type, ast.FieldType):diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.pyindex f7ebf6d84ced..f3ed8dc84997 100644--- a/posthog/hogql/transforms/lazy_tables.py+++ b/posthog/hogql/transforms/lazy_tables.py@@ -532,6 +532,23 @@ class LazyTableResolver(TraversingVisitor):# Field types retain the original tables after the scope replaces them with subqueries.original_select_type = ast.SelectQueryType(tables=dict(select_type.tables))+ # Virtual-table fields take their own export name, which must not equal the name of any other export.+ export_names: dict[int, str] = {}++ def allocate_export_name(+ accessed: dict[str, list[str | int]], field: ast.FieldType, chain: list[str | int]+ ) -> str:+ if not isinstance(field.table_type, ast.VirtualTableType):+ return field.name+ base = "___".join(str(link) for link in chain)+ name = base+ counter = 1+ while name in accessed and accessed[name] != chain:+ counter += 1+ name = f"{base}_{counter}"+ export_names[id(field)] = name+ return name+# First properties, then fields. This way we always get the smallest units to query first.matched_properties: list[ast.PropertyType | ast.FieldType] = [property for property in field_collector if isinstance(property, ast.PropertyType)@@ -539,6 +556,10 @@ class LazyTableResolver(TraversingVisitor):matched_fields: list[ast.PropertyType | ast.FieldType] = [field for field in field_collector if isinstance(field, ast.FieldType)]+ # Virtual-table fields go last, so each one picks a name that no other export has taken.+ matched_fields.sort(+ key=lambda field: isinstance(field, ast.FieldType) and isinstance(field.table_type, ast.VirtualTableType)+ )sorted_properties: list[ast.PropertyType | ast.FieldType] = matched_properties + matched_fields# Look for tables without requested fields to support cases like `select count() from table`@@ -646,7 +667,9 @@ class LazyTableResolver(TraversingVisitor):property.joined_subquery_field_name = "___".join(str(x) for x in chain)new_join.fields_accessed[property.joined_subquery_field_name] = chainelse:- new_join.fields_accessed[field.name] = chain+ new_join.fields_accessed[allocate_export_name(new_join.fields_accessed, field, chain)] = (+ chain+ )elif isinstance(table_type, ast.LazyTableType):table_name = get_long_table_name(select_type, table_type)if table_name not in tables_to_add:@@ -667,7 +690,9 @@ class LazyTableResolver(TraversingVisitor):property.joined_subquery_field_name = "___".join(str(x) for x in lazy_chain)new_table.fields_accessed[property.joined_subquery_field_name] = lazy_chainelse:- new_table.fields_accessed[field.name] = lazy_chain+ new_table.fields_accessed[+ allocate_export_name(new_table.fields_accessed, field, lazy_chain)+ ] = lazy_chainelif isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):# Widen mypy's view of the inner type: the annotated union is# `TableType | LazyTableType`, but the LazyJoinType branch below is a@@ -701,7 +726,9 @@ class LazyTableResolver(TraversingVisitor):property.joined_subquery_field_name = "___".join(str(x) for x in field_chain)new_join.fields_accessed[property.joined_subquery_field_name] = field_chainelse:- new_join.fields_accessed[field.name] = field_chain+ new_join.fields_accessed[+ allocate_export_name(new_join.fields_accessed, field, field_chain)+ ] = field_chainelif isinstance(inner_table_type, ast.LazyTableType):table_name = get_long_table_name(select_type, table_type)if table_name not in tables_to_add:@@ -723,7 +750,9 @@ class LazyTableResolver(TraversingVisitor):property.joined_subquery_field_name = "___".join(str(x) for x in chain)new_table.fields_accessed[property.joined_subquery_field_name] = chainelse:- new_table.fields_accessed[field.name] = chain+ new_table.fields_accessed[+ allocate_export_name(new_table.fields_accessed, field, chain)+ ] = chain# Make sure we also add fields we will use for the join's "ON" condition into the list of fields accessed.# Without this "pdi.person.id" won't work if you did not ALSO select "pdi.person_id" explicitly for the join.@@ -952,6 +981,10 @@ class LazyTableResolver(TraversingVisitor):continueif isinstance(field_or_property, ast.FieldType):+ # A field still on its virtual table reads the export name allocated above.+ # A field that a wrapper subquery already rebound keeps the name the wrapper gave it.+ if isinstance(field_or_property.table_type, ast.VirtualTableType):+ field_or_property.name = export_names.get(id(field_or_property), field_or_property.name)field_or_property.table_type = table_typeelif isinstance(field_or_property, ast.PropertyType):field_or_property.field_type.table_type = table_typediff --git a/posthog/hogql/transforms/test/test_flag_calls_union.py b/posthog/hogql/transforms/test/test_flag_calls_union.pyindex 4a2d55c649d9..615da6e38e91 100644--- a/posthog/hogql/transforms/test/test_flag_calls_union.py+++ b/posthog/hogql/transforms/test/test_flag_calls_union.py@@ -1,4 +1,5 @@import re+import jsonimport uuidfrom datetime import UTC, datetime, timedeltafrom typing import Any, cast@@ -657,15 +658,14 @@ class TestReadFlagCallsWithUnion(BaseTest):False,mode=ONLY,),- # Each scan exports one created_at, so the two fields never share a column.+ # Each scan exports its own property columns, so paths that join to one name in separate scans never share a column.param(- "fields_of_one_name_in_separate_scans",- f"SELECT created_at FROM events WHERE {MIXED} UNION ALL "- f"SELECT person.created_at FROM events WHERE {MIXED}",+ "property_paths_of_one_name_in_separate_scans",+ f"SELECT properties.a.b FROM events WHERE {MIXED} UNION ALL "+ f"SELECT properties.a___b FROM events WHERE {MIXED}",True,True,mode=ONLY,- persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,),param("alias_of_event_in_a_mixed_filter_on_mode_2",@@ -778,21 +778,11 @@ class TestReadFlagCallsWithUnion(BaseTest):"probe_join",warehouse_join=True,),- (- "field_and_virtual_field_of_one_name",- f"SELECT created_at, person.created_at FROM events WHERE {MIXED}",- "created_at and poe.created_at",- ),("properties_whose_paths_join_to_one_name",f"SELECT properties.a.b, properties.a___b FROM events WHERE {MIXED}","properties.a.b and properties.a___b",),- (- "properties_and_virtual_properties",- f"SELECT properties, person.properties FROM events WHERE {MIXED}",- "properties and poe.properties",- ),param("property_in_a_typed_materialized_column",f"SELECT properties.probe FROM events WHERE {MIXED}",@@ -1578,6 +1568,31 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):assert response.results == [("$feature_flag_called", "flag_evaluations", 12.5), ("$pageview", None, 20.0)]+ @parameterized.expand(+ [+ ("mode_1", FlagEvaluationsMode.READ_FLAG_EVALUATIONS),+ ("mode_2", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY),+ ]+ )+ def test_fields_of_one_name_on_events_and_person_return_their_own_values(+ self, _name: str, mode: FlagEvaluationsMode+ ):+ response = self._run(+ f"SELECT event, properties.source, person.properties.email, properties, person.properties, created_at, "+ f"person.created_at FROM events WHERE {MIXED} AND distinct_id = 'recent' "+ "AND timestamp >= now() - INTERVAL 5 DAY ORDER BY event",+ mode,+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,+ )++ flag_call, pageview = response.results+ assert flag_call[:3] == ("$feature_flag_called", "flag_evaluations", "[email protected]")+ assert json.loads(flag_call[3])["source"] == "flag_evaluations"+ assert json.loads(flag_call[4]) == {"email": "[email protected]"}+ assert pageview[:3] == ("$pageview", None, "[email protected]")+ assert json.loads(pageview[4]) == {"email": "[email protected]"}+ assert all(row[6] is not None and row[5] != row[6] for row in response.results)+@parameterized.expand([("read_flag_evaluations", READ), ("flag_evaluations_only", ONLY)])def test_json_extract_string_reads_the_raw_json_value(self, _name: str, mode: FlagEvaluationsMode):for name, kind in (@@ -1782,16 +1797,7 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):} <= set(accepted)types: dict[bool, dict[str, str]] = {False: {}, True: {}}# One query per few fields keeps each union within the memory of a local ClickHouse.- # Two fields of one name, such as goe_0.key and goe_1.key, would return one column, so they go in different queries.- batches: list[list[str]] = []- for name in accepted:- batch = next(- (b for b in batches if len(b) < 8 and all(n.rsplit(".", 1)[-1] != name.rsplit(".", 1)[-1] for n in b)),- None,- )- if batch is None:- batches.append(batch := [])- batch.append(name)+ batches = [accepted[i : i + 8] for i in range(0, len(accepted), 8)]for names in batches:types_query = f"SELECT {', '.join(f'toTypeName(any({name}))' for name in names)} FROM events WHERE {MIXED}"for union in (False, True):diff --git a/posthog/hogql/transforms/test/test_lazy_tables.py b/posthog/hogql/transforms/test/test_lazy_tables.pyindex 9653dd05eeb5..45f9004feedc 100644--- a/posthog/hogql/transforms/test/test_lazy_tables.py+++ b/posthog/hogql/transforms/test/test_lazy_tables.py@@ -29,7 +29,14 @@ from posthog.hogql.database.database import (_use_person_properties_from_events,)from posthog.hogql.database.lazy_join_tags import PERSONS-from posthog.hogql.database.models import LazyJoin, VirtualTable+from posthog.hogql.database.models import (+ IntegerDatabaseField,+ LazyJoin,+ LazyTable,+ LazyTableToAdd,+ TableNode,+ VirtualTable,+)from posthog.hogql.database.schema.traffic_type import create_is_bot_fieldfrom posthog.hogql.parser import parse_selectfrom posthog.hogql.printer import prepare_and_print_ast@@ -164,6 +171,70 @@ BUILT_IN_FIELDS = ()+class _ProbeLazyTable(LazyTable):+ fields: dict = {+ "x": IntegerDatabaseField(name="x"),+ "v___x": IntegerDatabaseField(name="v___x"),+ "v": VirtualTable(fields={"x": IntegerDatabaseField(name="x")}),+ }++ def lazy_select(self, table_to_add: LazyTableToAdd, context, node):+ return ast.SelectQuery(+ select=[ast.Alias(alias=name, expr=ast.Constant(value=name)) for name in table_to_add.fields_accessed]+ )++ def to_printed_clickhouse(self, context):+ return "probe"++ def to_printed_hogql(self):+ return "probe"+++class TestVirtualFieldExports(SimpleTestCase):+ @parameterized.expand([("bare", "probe", "probe"), ("aliased", "probe AS p", "p")])+ def test_virtual_field_export_never_takes_the_name_of_another_export(+ self, _name: str, source: str, prefix: str+ ) -> None:+ database = Database()+ database.tables.children["probe"] = TableNode(name="probe", table=_ProbeLazyTable())+ context = HogQLContext(+ team_id=1,+ team=Team(id=1, organization=Organization()),+ database=database,+ enable_select_queries=True,+ use_new_events_schema=False,+ )++ with patch("posthog.hogql.transforms.property_types.load_property_metadata", return_value=PropertyMetadata()):+ printed, _ = prepare_and_print_ast(+ parse_select(f"SELECT {prefix}.x, {prefix}.v.x, {prefix}.v___x FROM {source}"), context, "clickhouse"+ )++ values = re.findall(r"%\(hogql_val_\d+\)s AS (`?\w+`?)", printed)+ assert sorted(name.strip("`") for name in values) == ["v___x", "v___x_2", "x"]+ assert printed.startswith(f"SELECT {prefix}.x, {prefix}.v___x_2 AS x, {prefix}.v___x AS v___x FROM")++ def test_virtual_field_read_through_a_wrapped_lazy_join_names_the_column_the_wrapper_exports(self) -> None:+ context = HogQLContext(+ team_id=1,+ team=Team(id=1, organization=Organization()),+ database=Database(),+ enable_select_queries=True,+ apply_events_retention_floor=False,+ use_new_events_schema=False,+ )+ query = (+ "SELECT r.events.poe.created_at, r.events.created_at FROM events AS e "+ "JOIN raw_session_replay_events AS r ON r.events.event = e.event"+ )++ with patch("posthog.hogql.transforms.property_types.load_property_metadata", return_value=PropertyMetadata()):+ printed, _ = prepare_and_print_ast(parse_select(query), context, "clickhouse")++ assert printed.startswith("SELECT r.events___poe___created_at, r.events___created_at AS created_at FROM")+ assert "AS events___poe___created_at" in printed++class TestWrappedEventJoins(SimpleTestCase):@parameterized.expand([("legacy", False), ("native", True)])def test_reads_exported_columns(self, _name: str, native: bool) -> None: