Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): correct flag-call union reads of old calls and JSON extracts (#129825)

Phil Haack1d0a33f55c5c
diff --git a/posthog/hogql/database/schema/events_with_flag_calls.py b/posthog/hogql/database/schema/events_with_flag_calls.py
index fbebff4c70cb..bbf6bb74e023 100644
--- a/posthog/hogql/database/schema/events_with_flag_calls.py
+++ b/posthog/hogql/database/schema/events_with_flag_calls.py
@@ -125,7 +125,8 @@ class FlagEvaluationsColumns:
class EventsWithFlagCallsTable(LazyTable):
- """`events` with its flag calls since the coverage start read from `posthog.flag_evaluations` instead.
+ """`events` with its keyed flag calls read from `posthog.flag_evaluations` instead: since the coverage start on mode
+ 1, and every call that table retains on mode 2.
Its `fields` is the events table's own dict, so every name binds as it does on `events`.
"""
@@ -203,7 +204,11 @@ class EventsWithFlagCallsTable(LazyTable):
)
# On mode 2 every flag call comes from flag_evaluations, as the flag_evaluations swap reads it. That leaves out
# the calls without a key, which ingestion keeps writing to events.
- moved = flag_call if only_in_flag_evaluations else flag_call_since_cutoff
+ moved: ast.Expr = flag_call
+ flag_evaluations_since_cutoff: ast.Expr | None = None
+ if not only_in_flag_evaluations:
+ moved = flag_call_since_cutoff
+ flag_evaluations_since_cutoff = parse_expr("flag_evaluations.timestamp >= {cutoff}", {"cutoff": cutoff})
branches = [events_branch(ast.Not(expr=moved))]
if not only_in_flag_evaluations and filters.keyless_flag_calls_can_match:
branches.append(events_branch(flag_call_since_cutoff, ast.Not(expr=parse_expr(_HAS_FLAG_KEY))))
@@ -215,7 +220,7 @@ class EventsWithFlagCallsTable(LazyTable):
),
where=and_conditions(
[
- parse_expr("flag_evaluations.timestamp >= {cutoff}", {"cutoff": cutoff}),
+ flag_evaluations_since_cutoff,
filters.flag_evaluations,
sample_flag_evaluations_filter(sample, flag_evaluations_field(["distinct_id"]))
if sample is not None
diff --git a/posthog/hogql/transforms/json_property_pushdown.py b/posthog/hogql/transforms/json_property_pushdown.py
index 750a0cb89573..8702970bd85c 100644
--- a/posthog/hogql/transforms/json_property_pushdown.py
+++ b/posthog/hogql/transforms/json_property_pushdown.py
@@ -1,6 +1,7 @@
from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.models import StringJSONDatabaseField
+from posthog.hogql.database.schema.events_with_flag_calls import union_table_of
from posthog.hogql.visitor import CloningVisitor, TraversingVisitor
# JSON-string extraction calls that are equivalent to a HogQL string property access.
@@ -36,6 +37,11 @@ def _matched_property_access(node: ast.AST, context: HogQLContext):
table_type = table_type.table_type
if not isinstance(table_type, (ast.LazyTableType, ast.LazyJoinType)):
return None
+ # On the union, property access applies event property types and decodes escapes unlike JSONExtractString. The
+ # union also validated its reads before this rewrite. A join to events keeps the rewrite, because its whole-field
+ # reads of `properties` and `person.properties` share one export name.
+ if union_table_of(table_type) is not None:
+ return None
if not isinstance(field_type.resolve_database_field(context), StringJSONDatabaseField):
return None
diff --git a/posthog/hogql/transforms/test/test_flag_calls_union.py b/posthog/hogql/transforms/test/test_flag_calls_union.py
index f99587ac7be2..1b0501158285 100644
--- a/posthog/hogql/transforms/test/test_flag_calls_union.py
+++ b/posthog/hogql/transforms/test/test_flag_calls_union.py
@@ -32,6 +32,7 @@ from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
from posthog.hogql.database.models import DatabaseField, FieldTraverser, LazyJoin, StringJSONDatabaseField, VirtualTable
+from posthog.hogql.database.schema.flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYS
from posthog.hogql.database.schema.util.where_clause_extractor import is_nondeterministic_function
from posthog.hogql.errors import QueryError
from posthog.hogql.functions.mapping import HOGQL_CLICKHOUSE_FUNCTIONS
@@ -855,15 +856,6 @@ class TestReadFlagCallsWithUnion(BaseTest):
"WHERE in(e.event",
"WHERE in(event, tuple('$pageview', '$feature_flag_called')) LIMIT 10) AS e LEFT JOIN",
),
- (
- "json_extract_of_a_materialized_key_reads_its_column",
- f"SELECT JSONExtractString(properties, 'mykey') FROM events WHERE {MIXED}",
- PersonsOnEventsMode.DISABLED,
- {},
- "events.mat_mykey",
- "events.properties, 'mykey'",
- None,
- ),
]
)
def test_union_scan_keeps_the_pushdown_that_events_gets(
@@ -1108,6 +1100,29 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):
assert rows[:4] == [*history, *recent]
assert [row[:3] for row in rows[4:]] == keyless
+ @parameterized.expand([("read_flag_evaluations", READ), ("flag_evaluations_only", ONLY)])
+ def test_flag_only_and_mixed_queries_count_a_call_from_before_the_coverage_start(
+ self, _name: str, mode: FlagEvaluationsMode
+ ):
+ # Mode 2 writes a flag call only to flag_evaluations.
+ timestamp = datetime.now(UTC) - timedelta(days=FLAG_EVALUATIONS_COMPLETE_DAYS + 1)
+ if mode == READ:
+ _create_event(
+ event="$feature_flag_called",
+ distinct_id="edge",
+ team=self.team,
+ timestamp=timestamp,
+ properties={"$feature_flag": "edge-flag"},
+ )
+ flush_persons_and_events()
+ _create_flag_evaluations(self.team.pk, "edge-flag", timestamp=timestamp, distinct_id="edge")
+ bound = f"distinct_id = 'edge' AND timestamp >= now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS + 3} DAY"
+
+ flag_only = self._run(f"SELECT count() FROM events WHERE event = '$feature_flag_called' AND {bound}", mode)
+ mixed = self._run(f"SELECT countIf(event = '$feature_flag_called') FROM events WHERE {MIXED} AND {bound}", mode)
+
+ assert (flag_only.results, mixed.results) == ([(1,)], [(1,)])
+
def test_inline_cohort_counts_flag_calls_that_only_flag_evaluations_holds(self):
fresh = _create_person(distinct_ids=["fresh"], team_id=self.team.pk)
_create_flag_evaluations(
@@ -1251,6 +1266,59 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):
assert response.results == [("$feature_flag_called", "flag_evaluations", 12.5), ("$pageview", None, 20.0)]
+ @parameterized.expand([("read_flag_evaluations", READ), ("flag_evaluations_only", ONLY)])
+ def test_json_extract_string_reads_the_raw_json_value(self, _name: str, mode: FlagEvaluationsMode):
+ for name, kind in (
+ ("probe", PropertyDefinition.Type.EVENT),
+ ("amount", PropertyDefinition.Type.EVENT),
+ ("probe", PropertyDefinition.Type.PERSON),
+ ):
+ PropertyDefinition.objects.create(team=self.team, name=name, property_type="Numeric", type=kind)
+ now = datetime.now(UTC)
+ person = _create_person(distinct_ids=["probe"], team_id=self.team.pk)
+ properties = {"probe": "text", "amount": "12.5", "newline": "a\nb", "a": {"b": "nested"}, "a___b": "flat"}
+ for event in ("$pageview", "$feature_flag_called"):
+ _create_event(
+ event=event,
+ distinct_id="probe",
+ person_id=person.uuid,
+ person_properties={"probe": "text"},
+ team=self.team,
+ timestamp=now - timedelta(days=1),
+ properties={"$feature_flag": "probe-flag", "source": "events", **properties},
+ )
+ _create_flag_evaluations(
+ self.team.pk,
+ "probe-flag",
+ timestamp=now - timedelta(days=1),
+ distinct_id="probe",
+ person_id=person.uuid,
+ person_properties={"probe": "text"},
+ properties={"source": "flag_evaluations", **properties},
+ )
+ flush_persons_and_events()
+
+ def run(alias: str, expressions: str):
+ return self._run(
+ f"SELECT {alias}.event, {alias}.properties.source, {expressions} FROM events AS {alias} "
+ f"WHERE {alias}.{MIXED} AND {alias}.distinct_id = 'probe' ORDER BY {alias}.event",
+ mode,
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,
+ ).results
+
+ def rows(*values) -> list[tuple]:
+ return [("$feature_flag_called", "flag_evaluations", *values), ("$pageview", "events", *values)]
+
+ # A rewritten read of the flat key `a___b` gets the export name of `properties.a.b`.
+ assert run(
+ "events",
+ "JSONExtractString(properties, 'probe'), toFloatOrZero(JSONExtractString(properties, 'amount')), "
+ "JSONExtractString(properties, 'newline'), properties.a.b, JSONExtractString(properties, 'a___b')",
+ ) == rows("text", 12.5, "a\nb", "nested", "flat")
+ assert run("e", "JSONExtractString(e.properties, 'probe')") == rows("text")
+ # The union refuses a query that reads both `properties` and `person.properties`.
+ assert run("events", "JSONExtractString(person.properties, 'probe')") == rows("text")
+
def test_prewhere_reads_both_tables(self):
response = self._run(
f"SELECT count() FROM events PREWHERE {MIXED} WHERE timestamp >= now() - INTERVAL 120 DAY",
diff --git a/posthog/hogql/transforms/test/test_json_property_pushdown.py b/posthog/hogql/transforms/test/test_json_property_pushdown.py
index 91f920469010..fb40b1d58760 100644
--- a/posthog/hogql/transforms/test/test_json_property_pushdown.py
+++ b/posthog/hogql/transforms/test/test_json_property_pushdown.py
@@ -2,14 +2,16 @@ from posthog.test.base import BaseTest
from parameterized import parameterized
+from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
+
from posthog.hogql.context import HogQLContext
from posthog.hogql.parser import parse_select
from posthog.hogql.printer import prepare_and_print_ast
class TestJSONPropertyPushdown(BaseTest):
- def _print(self, query: str) -> str:
- context = HogQLContext(team_id=self.team.pk, enable_select_queries=True)
+ def _print(self, query: str, **context_options) -> str:
+ context = HogQLContext(team_id=self.team.pk, enable_select_queries=True, **context_options)
printed, _ = prepare_and_print_ast(parse_select(query), context, dialect="clickhouse")
return printed
@@ -24,6 +26,21 @@ class TestJSONPropertyPushdown(BaseTest):
wrapped = self._print(f"SELECT ifNull(properties.name, '') AS name FROM {table}")
self.assertEqual(extract, wrapped)
+ def test_json_extract_string_on_a_join_to_groups_is_rewritten(self):
+ printed = self._print("SELECT JSONExtractString(group_0.properties, 'probe') FROM events")
+ self.assertIn("properties___probe", printed)
+
+ def test_event_and_person_reads_through_a_join_to_events_read_their_own_columns(self):
+ printed = self._print(
+ "SELECT JSONExtractString(events.properties, 'label'), "
+ "JSONExtractString(events.person.properties, 'label') FROM raw_session_replay_events",
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
+ ),
+ )
+ self.assertIn(" AS properties___label,", printed)
+ self.assertIn(" AS poe___properties___label,", printed)
+
def test_whole_properties_access_still_aggregates_blob(self):
printed = self._print("SELECT key, properties FROM groups")
self.assertIn("argMax(groups.group_properties,", printed)