:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(endpoints): coerce materialized variable filters to the column type (#122776)
Jovan Sakovic1016b16426e3
diff --git a/products/endpoints/backend/logic/strategies.py b/products/endpoints/backend/logic/strategies.pyindex 91f6bd14348c..f3f611f6f97c 100644--- a/products/endpoints/backend/logic/strategies.py+++ b/products/endpoints/backend/logic/strategies.py@@ -12,9 +12,12 @@ Everything that branches on ``query["kind"]`` belongs here, behind the sharedstay kind-agnostic."""+import reimport abc+import mathimport dataclassesfrom datetime import datetime+from decimal import Decimal, InvalidOperationfrom functools import cached_propertyfrom typing import TYPE_CHECKING, ClassVar, Optional@@ -71,20 +74,59 @@ def add_where_condition(select_query: ast.SelectQuery, condition: ast.Expr) -> Nselect_query.where = condition+def coerce_filter_value(value: object, column_type: str | None) -> object:+ if value is None or isinstance(value, bool) or not column_type:+ return value++ while column_type.endswith(")"):+ for wrapper in ("Nullable(", "LowCardinality("):+ if column_type.startswith(wrapper):+ column_type = column_type[len(wrapper) : -1]+ break+ else:+ break++ if column_type == "String" or column_type.startswith("FixedString("):+ return str(value)++ if not isinstance(value, str):+ return value++ integer_type = re.fullmatch(r"(U?)Int(8|16|32|64|128|256)", column_type)+ if integer_type and re.fullmatch(r"[+-]?\d+", value):+ integer_value = int(value)+ bits = int(integer_type.group(2))+ minimum = 0 if integer_type.group(1) else -(2 ** (bits - 1))+ maximum = 2**bits - 1 if integer_type.group(1) else 2 ** (bits - 1) - 1+ return integer_value if minimum <= integer_value <= maximum else value++ if column_type in {"Float32", "Float64"}:+ try:+ float_value = float(value)+ if math.isfinite(float_value) and Decimal(value) == Decimal(str(float_value)):+ return float_value+ except (InvalidOperation, ValueError, OverflowError):+ pass++ return value++def apply_where_filter(select_query: ast.SelectQuery,column: str,- value: str,+ value: object,op: ast.CompareOperationOp = ast.CompareOperationOp.Eq,value_wrapper_fns: list[str] | None = None,bucket_fn: str | None = None,+ column_type: str | None = None,) -> None:"""Add a comparison filter to WHERE clause.When bucket_fn is set (range variables), wrap the filter value with thebucket function so the comparison matches the bucketed column values."""- right_expr: ast.Expr = ast.Constant(value=value)+ constant_value = value if value_wrapper_fns or bucket_fn else coerce_filter_value(value, column_type)+ right_expr: ast.Expr = ast.Constant(value=constant_value)for fn in reversed(value_wrapper_fns or []):right_expr = ast.Call(name=fn, args=[right_expr])if bucket_fn:@@ -541,6 +583,11 @@ class HogQLEndpointStrategy(EndpointQueryStrategy):for mat_var in self.materialized_variables:var_value = data.variables.get(mat_var.code_name)if var_value is not None:+ column_type = (+ self.version.saved_query.get_clickhouse_column_type(mat_var.code_name)+ if self.version.saved_query+ else None+ )apply_where_filter(select_query,mat_var.code_name,@@ -548,6 +595,7 @@ class HogQLEndpointStrategy(EndpointQueryStrategy):op=mat_var.operator,value_wrapper_fns=mat_var.value_wrapper_fns,bucket_fn=mat_var.bucket_fn,+ column_type=column_type,)return Nonediff --git a/products/endpoints/backend/tests/test_endpoint_execution.py b/products/endpoints/backend/tests/test_endpoint_execution.pyindex 66474c803abc..f690185d776e 100644--- a/products/endpoints/backend/tests/test_endpoint_execution.py+++ b/products/endpoints/backend/tests/test_endpoint_execution.py@@ -619,43 +619,85 @@ class TestEndpointExecution(ClickhouseTestMixin, APIBaseTest):# MATERIALIZED HOGQL ENDPOINTS# =========================================================================- def test_materialized_hogql_endpoint_filters_by_variable(self):+ @parameterized.expand(+ [+ ("number", 424242),+ ("string", "424242"),+ ]+ )+ def test_materialized_hogql_endpoint_filters_by_variable_type(self, _name: str, capture_id: object):+ stored_capture_id = "424242"+ _create_event(+ event="capture",+ distinct_id="user1",+ team=self.team,+ timestamp="2026-01-05 12:00:00",+ properties={"capture_id": stored_capture_id},+ )+ flush_persons_and_events()+ capture_id_var = InsightVariable.objects.create(+ team=self.team,+ name="Capture ID",+ code_name="capture_id",+ type=InsightVariable.Type.STRING,+ default_value=stored_capture_id,+ )+endpoint = create_endpoint_with_version(- name="mat_hogql_var",+ name=f"mat-hogql-var-{_name}",team=self.team,query={"kind": "HogQLQuery",- "query": "SELECT count() FROM events WHERE event = {variables.event_name}",+ "query": "SELECT count() FROM events WHERE properties.capture_id = {variables.capture_id}","variables": {- str(self.event_name_var.id): {- "variableId": str(self.event_name_var.id),- "code_name": "event_name",- "value": "$pageview",+ str(capture_id_var.id): {+ "variableId": str(capture_id_var.id),+ "code_name": "capture_id",+ "value": stored_capture_id,}},},created_by=self.user,is_active=True,)- self._materialize_endpoint(endpoint)+ inline_response = self.client.post(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/run/",+ {"variables": {"capture_id": stored_capture_id}},+ format="json",+ )+ self.assertEqual(inline_response.status_code, status.HTTP_200_OK, inline_response.json())- # Execute with variable filter - should filter materialized table by event_name- with mock.patch.object(- EndpointExecutionService, "_execute_query_and_respond", return_value=Response({})- ) as mock_exec:+ saved_query = self._materialize_endpoint(endpoint)+ saved_query.columns = {+ "count()": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt64", "valid": True},+ "capture_id": {"hogql": "StringDatabaseField", "clickhouse": "String", "valid": True},+ }+ saved_query.save(update_fields=["columns"])++ def run_query(_team, query, **_kwargs):+ if (+ query.modifiers+ and query.modifiers.useMaterializedViews+ and stored_capture_id in query.query+ and f"'{stored_capture_id}'" not in query.query+ ):+ raise CHQueryErrorNoCommonType(+ "There is no supertype for types String, UInt64",+ code=386,+ code_name="no_common_type",+ )+ return {"results": inline_response.json()["results"], "columns": ["count()"]}++ with mock.patch("products.endpoints.backend.logic.execution.process_query_model", side_effect=run_query):response = self.client.post(f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/run/",- {"variables": {"event_name": "$pageleave"}},+ {"variables": {"capture_id": capture_id}},format="json",)- self.assertEqual(response.status_code, status.HTTP_200_OK)- mock_exec.assert_called()- # Verify the query has WHERE clause filtering by event_name- query_request_data = mock_exec.call_args[0][0]- query_sql = query_request_data["query"]["query"].lower()- self.assertIn("event_name", query_sql)- self.assertIn("$pageleave", query_sql)+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertIs(response.json()["endpoint_materialized"], True)+ self.assertEqual(response.json()["results"], inline_response.json()["results"])def test_materialized_hogql_endpoint_selects_only_original_columns(self):endpoint = create_endpoint_with_version(diff --git a/products/endpoints/backend/tests/test_materialization_analysis.py b/products/endpoints/backend/tests/test_materialization_analysis.pyindex 1f8325b77a18..f8c873c34e9b 100644--- a/products/endpoints/backend/tests/test_materialization_analysis.py+++ b/products/endpoints/backend/tests/test_materialization_analysis.py@@ -581,7 +581,9 @@ class TestCombinatorReaggregation(SimpleTestCase):class TestMaterializedReadPath(SimpleTestCase):"""Test that the read path applies value_wrapper_fns when filtering the materialized table."""- def _build_read_query(self, query_str: str, variables_meta: dict, variable_values: dict) -> str:+ def _build_read_query(+ self, query_str: str, variables_meta: dict, variable_values: dict, column_type: str | None = None+ ) -> str:"""Simulate the materialized read path: analyze variables, then build a SELECT with filters."""hogql_query = {"kind": "HogQLQuery", "query": query_str, "variables": variables_meta}_, _, var_infos = analyze_variables_for_materialization(hogql_query)@@ -601,6 +603,7 @@ class TestMaterializedReadPath(SimpleTestCase):op=mat_var.operator,value_wrapper_fns=mat_var.value_wrapper_fns,bucket_fn=mat_var.bucket_fn,+ column_type=column_type,)return select_query.to_hogql()@@ -625,14 +628,35 @@ class TestMaterializedReadPath(SimpleTestCase):assert "toDate('2024-01-15 14:30:00')" in result- def test_lower_wrapper_applied_to_value(self):+ @parameterized.expand(+ [+ (+ "lower",+ "SELECT count() FROM events WHERE lower(event) = lower({variables.value})",+ "$PageView",+ None,+ "lower('$PageView')",+ ),+ (+ "city_hash_with_numeric_column",+ "SELECT count() FROM events WHERE cityHash64(distinct_id) = cityHash64({variables.value})",+ "123",+ "UInt64",+ "cityHash64('123')",+ ),+ ]+ )+ def test_wrapper_applied_to_value(+ self, _name: str, query: str, value: str, column_type: str | None, expected: str+ ) -> None:result = self._build_read_query(- "SELECT count() FROM events WHERE lower(event) = lower({variables.event_name})",- {"var-1": {"code_name": "event_name", "value": "$PageView"}},- {"event_name": "$PageView"},+ query,+ {"var-1": {"code_name": "value", "value": value}},+ {"value": value},+ column_type,)- assert "lower('$PageView')" in result+ assert expected in resultdef test_range_with_wrapper_both_sides(self):result = self._build_read_query(