:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(endpoints): filter scalar breakdown values on materialized reads (#122909)
Jovan Sakovic425e8dd88e94
diff --git a/products/endpoints/backend/logic/strategies.py b/products/endpoints/backend/logic/strategies.pyindex f3f611f6f97c..b8731d0097f9 100644--- a/products/endpoints/backend/logic/strategies.py+++ b/products/endpoints/backend/logic/strategies.py@@ -175,16 +175,30 @@ def get_breakdown_infos(breakdown_filter: dict) -> list[tuple[str, str]]:return list(iter_breakdowns(breakdown_filter))-def build_breakdown_filter_condition(query_kind: str | None, value: str, array_index: int = 0) -> ast.Expr | None:+def breakdown_value_is_array(query_kind: str | None, breakdown_filter: dict) -> bool:+ """Return whether the materialized `breakdown_value` column is an Array.++ RetentionQuery always converts its breakdown to the `breakdowns` form, so it stores an Array.+ TrendsQuery stores an Array only for `breakdowns`; the single `breakdown` form stores a scalar.+ """+ if query_kind == "RetentionQuery":+ return True+ return bool(breakdown_filter.get("breakdowns"))+++def build_breakdown_filter_condition(+ query_kind: str | None, value: str, array_index: int = 0, *, is_array: bool = True+) -> ast.Expr | None:"""Build the appropriate WHERE condition for breakdown filtering based on query type.Different insight types store breakdowns in different columns:- - TrendsQuery, RetentionQuery: `breakdown_value` Array column+ - TrendsQuery, RetentionQuery: `breakdown_value` column, see `breakdown_value_is_array`- LifecycleQuery: No breakdown support- array_index controls how to access the breakdown column:- - 0: single breakdown — use has() on the full array- - 1+: multiple breakdowns — use equality on breakdown_value[N]+ is_array=False: scalar column, so use equality on breakdown_value.+ Otherwise array_index controls how to access the breakdown column:+ - 0: single breakdown, so use has() on the full array+ - 1+: multiple breakdowns, so use equality on breakdown_value[N]"""if query_kind not in BREAKDOWN_SUPPORTED_QUERY_TYPES:logger.warning(@@ -193,6 +207,13 @@ def build_breakdown_filter_condition(query_kind: str | None, value: str, array_i)return None+ if not is_array:+ return ast.CompareOperation(+ left=ast.Field(chain=["breakdown_value"]),+ op=ast.CompareOperationOp.Eq,+ right=ast.Constant(value=value),+ )+if array_index > 0:return ast.CompareOperation(left=ast.ArrayAccess(@@ -670,6 +691,10 @@ class InsightEndpointStrategy(EndpointQueryStrategy):def _breakdown_filter(self) -> dict:return self.query.get("breakdownFilter") or {}+ @property+ def _breakdown_value_is_array(self) -> bool:+ return breakdown_value_is_array(self.query_kind, self._breakdown_filter)+def allowed_variables(self, is_materialized: bool) -> set[str]:allowed: set[str] = set()@@ -709,7 +734,12 @@ class InsightEndpointStrategy(EndpointQueryStrategy):prop_value = getattr(prop, "value", None)if prop_value is not None:value = prop_value[0] if isinstance(prop_value, list) else prop_value- return build_breakdown_filter_condition(self.query_kind, str(value)) is not None+ return (+ build_breakdown_filter_condition(+ self.query_kind, str(value), is_array=self._breakdown_value_is_array+ )+ is not None+ )return Falsedef build_materialized_select(@@ -733,7 +763,9 @@ class InsightEndpointStrategy(EndpointQueryStrategy):if hasattr(prop, "key") and hasattr(prop, "value") and prop.value is not None:# Convert value to string for breakdown filtervalue = prop.value[0] if isinstance(prop.value, list) else prop.value- condition = build_breakdown_filter_condition(self.query_kind, str(value))+ condition = build_breakdown_filter_condition(+ self.query_kind, str(value), is_array=self._breakdown_value_is_array+ )if condition:add_where_condition(select_query, condition)# filters_override is deprecated — only the first property is used@@ -745,7 +777,10 @@ class InsightEndpointStrategy(EndpointQueryStrategy):for breakdown_prop, array_index in index_mapping.items():if breakdown_prop in data.variables:condition = build_breakdown_filter_condition(- self.query_kind, data.variables[breakdown_prop], array_index=array_index+ self.query_kind,+ data.variables[breakdown_prop],+ array_index=array_index,+ is_array=self._breakdown_value_is_array,)if condition:add_where_condition(select_query, condition)diff --git a/products/endpoints/backend/tests/test_endpoint_materialization.py b/products/endpoints/backend/tests/test_endpoint_materialization.pyindex fc04779c1f67..26511013f99e 100644--- a/products/endpoints/backend/tests/test_endpoint_materialization.py+++ b/products/endpoints/backend/tests/test_endpoint_materialization.py@@ -18,6 +18,7 @@ from rest_framework.exceptions import APIExceptionfrom rest_framework.response import Responsefrom posthog.hogql.errors import QueryError+from posthog.hogql.query import execute_hogql_queryfrom posthog.constants import RETENTION_FIRST_EVER_OCCURRENCE, TREND_FILTER_TYPE_EVENTSfrom posthog.models.activity_logging.activity_log import ActivityLog@@ -1396,10 +1397,86 @@ class TestEndpointMaterialization(ClickhouseTestMixin, APIBaseTest):query_sql = query_payload["query"].lower()# Should select from the materialized table nameself.assertIn("trends_breakdown_materialized", query_sql)- # Should filter by the breakdown value using has() for array column- self.assertIn("has(breakdown_value", query_sql)+ # A single `breakdown` stores breakdown_value as a scalar, so the filter is an equality+ self.assertIn("equals(breakdown_value, 'chrome')", query_sql)self.assertIn("chrome", query_sql)+ @parameterized.expand(+ [+ ("single_breakdown", {"breakdown": "$browser", "breakdown_type": "event"}, {"$browser": "Chrome"}),+ ("breakdowns_one", {"breakdowns": [{"property": "$browser", "type": "event"}]}, {"$browser": "Chrome"}),+ (+ "breakdowns_two",+ {"breakdowns": [{"property": "$browser", "type": "event"}, {"property": "$os", "type": "event"}]},+ {"$browser": "Chrome", "$os": "Mac OS X"},+ ),+ ]+ )+ def test_materialized_breakdown_filter_matches_stored_column_type(+ self, _name: str, breakdown_filter: dict[str, Any], variables: dict[str, str]+ ) -> None:+ for distinct_id, browser, os in [("u1", "Chrome", "Mac OS X"), ("u2", "Firefox", "Windows")]:+ _create_event(+ team=self.team,+ event="$pageview",+ distinct_id=distinct_id,+ properties={"$browser": browser, "$os": os},+ )+ flush_persons_and_events()++ endpoint = create_endpoint_with_version(+ name=f"breakdown_filter_{_name}",+ team=self.team,+ query=self._trends_query(breakdownFilter=breakdown_filter),+ created_by=self.user,+ is_active=True,+ )+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"is_materialized": True, "data_freshness_seconds": 43200},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())++ saved_query = endpoint.versions.get().saved_query+ assert saved_query is not None+ saved_query.status = DataWarehouseSavedQuery.Status.COMPLETED+ saved_query.last_run_at = timezone.now()+ saved_query.table = DataWarehouseTable.objects.create(+ team=self.team,+ name=saved_query.name,+ format=DataWarehouseTable.TableFormat.Parquet,+ url_pattern="s3://test-bucket/breakdown-filter",+ )+ saved_query.save()++ with mock.patch.object(+ EndpointExecutionService, "_execute_query_and_respond", return_value=Response({})+ ) as mock_exec:+ response = self.client.post(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/run",+ {"variables": variables},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ materialized_sql = mock_exec.call_args[0][0]["query"]["query"]++ # Swap the table for the HogQL it materializes, so the filter meets the real breakdown_value type.+ self.assertIn(f"FROM {saved_query.name} ", materialized_sql)+ result = execute_hogql_query(+ materialized_sql.replace(f"FROM {saved_query.name} ", f"FROM ({saved_query.query['query']}) "),+ team=self.team,+ )++ assert result.columns is not None+ breakdown_index = result.columns.index("breakdown_value")+ breakdown_values = [row[breakdown_index] for row in result.results]+ expected = list(variables.values())+ self.assertEqual(+ breakdown_values,+ [expected[0] if "breakdown" in breakdown_filter else expected],+ )+def test_materialized_insight_filters_override_without_value_does_not_bypass_required_breakdown(self):# SECURITY: a filters_override whose property carries no usable value adds no WHERE clause# in apply_materialized_filters, so it must not be allowed to skip the required-breakdown