Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(endpoints): filter scalar breakdown values on materialized reads (#122909)

Jovan Sakovic425e8dd88e94
diff --git a/products/endpoints/backend/logic/strategies.py b/products/endpoints/backend/logic/strategies.py
index f3f611f6f97c..b8731d0097f9 100644
--- a/products/endpoints/backend/logic/strategies.py
+++ b/products/endpoints/backend/logic/strategies.py
@@ -175,16 +175,30 @@ def get_breakdown_infos(breakdown_filter: dict) -> list[tuple[str, str]]:
return list(iter_breakdowns(breakdown_filter))
-def build_breakdown_filter_condition(query_kind: str | None, value: str, array_index: int = 0) -> ast.Expr | None:
+def breakdown_value_is_array(query_kind: str | None, breakdown_filter: dict) -> bool:
+ """Return whether the materialized `breakdown_value` column is an Array.
+
+ RetentionQuery always converts its breakdown to the `breakdowns` form, so it stores an Array.
+ TrendsQuery stores an Array only for `breakdowns`; the single `breakdown` form stores a scalar.
+ """
+ if query_kind == "RetentionQuery":
+ return True
+ return bool(breakdown_filter.get("breakdowns"))
+
+
+def build_breakdown_filter_condition(
+ query_kind: str | None, value: str, array_index: int = 0, *, is_array: bool = True
+) -> ast.Expr | None:
"""Build the appropriate WHERE condition for breakdown filtering based on query type.
Different insight types store breakdowns in different columns:
- - TrendsQuery, RetentionQuery: `breakdown_value` Array column
+ - TrendsQuery, RetentionQuery: `breakdown_value` column, see `breakdown_value_is_array`
- LifecycleQuery: No breakdown support
- array_index controls how to access the breakdown column:
- - 0: single breakdown — use has() on the full array
- - 1+: multiple breakdowns — use equality on breakdown_value[N]
+ is_array=False: scalar column, so use equality on breakdown_value.
+ Otherwise array_index controls how to access the breakdown column:
+ - 0: single breakdown, so use has() on the full array
+ - 1+: multiple breakdowns, so use equality on breakdown_value[N]
"""
if query_kind not in BREAKDOWN_SUPPORTED_QUERY_TYPES:
logger.warning(
@@ -193,6 +207,13 @@ def build_breakdown_filter_condition(query_kind: str | None, value: str, array_i
)
return None
+ if not is_array:
+ return ast.CompareOperation(
+ left=ast.Field(chain=["breakdown_value"]),
+ op=ast.CompareOperationOp.Eq,
+ right=ast.Constant(value=value),
+ )
+
if array_index > 0:
return ast.CompareOperation(
left=ast.ArrayAccess(
@@ -670,6 +691,10 @@ class InsightEndpointStrategy(EndpointQueryStrategy):
def _breakdown_filter(self) -> dict:
return self.query.get("breakdownFilter") or {}
+ @property
+ def _breakdown_value_is_array(self) -> bool:
+ return breakdown_value_is_array(self.query_kind, self._breakdown_filter)
+
def allowed_variables(self, is_materialized: bool) -> set[str]:
allowed: set[str] = set()
@@ -709,7 +734,12 @@ class InsightEndpointStrategy(EndpointQueryStrategy):
prop_value = getattr(prop, "value", None)
if prop_value is not None:
value = prop_value[0] if isinstance(prop_value, list) else prop_value
- return build_breakdown_filter_condition(self.query_kind, str(value)) is not None
+ return (
+ build_breakdown_filter_condition(
+ self.query_kind, str(value), is_array=self._breakdown_value_is_array
+ )
+ is not None
+ )
return False
def build_materialized_select(
@@ -733,7 +763,9 @@ class InsightEndpointStrategy(EndpointQueryStrategy):
if hasattr(prop, "key") and hasattr(prop, "value") and prop.value is not None:
# Convert value to string for breakdown filter
value = prop.value[0] if isinstance(prop.value, list) else prop.value
- condition = build_breakdown_filter_condition(self.query_kind, str(value))
+ condition = build_breakdown_filter_condition(
+ self.query_kind, str(value), is_array=self._breakdown_value_is_array
+ )
if condition:
add_where_condition(select_query, condition)
# filters_override is deprecated — only the first property is used
@@ -745,7 +777,10 @@ class InsightEndpointStrategy(EndpointQueryStrategy):
for breakdown_prop, array_index in index_mapping.items():
if breakdown_prop in data.variables:
condition = build_breakdown_filter_condition(
- self.query_kind, data.variables[breakdown_prop], array_index=array_index
+ self.query_kind,
+ data.variables[breakdown_prop],
+ array_index=array_index,
+ is_array=self._breakdown_value_is_array,
)
if condition:
add_where_condition(select_query, condition)
diff --git a/products/endpoints/backend/tests/test_endpoint_materialization.py b/products/endpoints/backend/tests/test_endpoint_materialization.py
index fc04779c1f67..26511013f99e 100644
--- a/products/endpoints/backend/tests/test_endpoint_materialization.py
+++ b/products/endpoints/backend/tests/test_endpoint_materialization.py
@@ -18,6 +18,7 @@ from rest_framework.exceptions import APIException
from rest_framework.response import Response
from posthog.hogql.errors import QueryError
+from posthog.hogql.query import execute_hogql_query
from posthog.constants import RETENTION_FIRST_EVER_OCCURRENCE, TREND_FILTER_TYPE_EVENTS
from posthog.models.activity_logging.activity_log import ActivityLog
@@ -1396,10 +1397,86 @@ class TestEndpointMaterialization(ClickhouseTestMixin, APIBaseTest):
query_sql = query_payload["query"].lower()
# Should select from the materialized table name
self.assertIn("trends_breakdown_materialized", query_sql)
- # Should filter by the breakdown value using has() for array column
- self.assertIn("has(breakdown_value", query_sql)
+ # A single `breakdown` stores breakdown_value as a scalar, so the filter is an equality
+ self.assertIn("equals(breakdown_value, 'chrome')", query_sql)
self.assertIn("chrome", query_sql)
+ @parameterized.expand(
+ [
+ ("single_breakdown", {"breakdown": "$browser", "breakdown_type": "event"}, {"$browser": "Chrome"}),
+ ("breakdowns_one", {"breakdowns": [{"property": "$browser", "type": "event"}]}, {"$browser": "Chrome"}),
+ (
+ "breakdowns_two",
+ {"breakdowns": [{"property": "$browser", "type": "event"}, {"property": "$os", "type": "event"}]},
+ {"$browser": "Chrome", "$os": "Mac OS X"},
+ ),
+ ]
+ )
+ def test_materialized_breakdown_filter_matches_stored_column_type(
+ self, _name: str, breakdown_filter: dict[str, Any], variables: dict[str, str]
+ ) -> None:
+ for distinct_id, browser, os in [("u1", "Chrome", "Mac OS X"), ("u2", "Firefox", "Windows")]:
+ _create_event(
+ team=self.team,
+ event="$pageview",
+ distinct_id=distinct_id,
+ properties={"$browser": browser, "$os": os},
+ )
+ flush_persons_and_events()
+
+ endpoint = create_endpoint_with_version(
+ name=f"breakdown_filter_{_name}",
+ team=self.team,
+ query=self._trends_query(breakdownFilter=breakdown_filter),
+ created_by=self.user,
+ is_active=True,
+ )
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"is_materialized": True, "data_freshness_seconds": 43200},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+
+ saved_query = endpoint.versions.get().saved_query
+ assert saved_query is not None
+ saved_query.status = DataWarehouseSavedQuery.Status.COMPLETED
+ saved_query.last_run_at = timezone.now()
+ saved_query.table = DataWarehouseTable.objects.create(
+ team=self.team,
+ name=saved_query.name,
+ format=DataWarehouseTable.TableFormat.Parquet,
+ url_pattern="s3://test-bucket/breakdown-filter",
+ )
+ saved_query.save()
+
+ with mock.patch.object(
+ EndpointExecutionService, "_execute_query_and_respond", return_value=Response({})
+ ) as mock_exec:
+ response = self.client.post(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/run",
+ {"variables": variables},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+ materialized_sql = mock_exec.call_args[0][0]["query"]["query"]
+
+ # Swap the table for the HogQL it materializes, so the filter meets the real breakdown_value type.
+ self.assertIn(f"FROM {saved_query.name} ", materialized_sql)
+ result = execute_hogql_query(
+ materialized_sql.replace(f"FROM {saved_query.name} ", f"FROM ({saved_query.query['query']}) "),
+ team=self.team,
+ )
+
+ assert result.columns is not None
+ breakdown_index = result.columns.index("breakdown_value")
+ breakdown_values = [row[breakdown_index] for row in result.results]
+ expected = list(variables.values())
+ self.assertEqual(
+ breakdown_values,
+ [expected[0] if "breakdown" in breakdown_filter else expected],
+ )
+
def test_materialized_insight_filters_override_without_value_does_not_bypass_required_breakdown(self):
# SECURITY: a filters_override whose property carries no usable value adds no WHERE clause
# in apply_materialized_filters, so it must not be allowed to skip the required-breakdown