Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(endpoints): coerce materialized variable filters to the column type (#122776)

Jovan Sakovic1016b16426e3
diff --git a/products/endpoints/backend/logic/strategies.py b/products/endpoints/backend/logic/strategies.py
index 91f6bd14348c..f3f611f6f97c 100644
--- a/products/endpoints/backend/logic/strategies.py
+++ b/products/endpoints/backend/logic/strategies.py
@@ -12,9 +12,12 @@ Everything that branches on ``query["kind"]`` belongs here, behind the shared
stay kind-agnostic.
"""
+import re
import abc
+import math
import dataclasses
from datetime import datetime
+from decimal import Decimal, InvalidOperation
from functools import cached_property
from typing import TYPE_CHECKING, ClassVar, Optional
@@ -71,20 +74,59 @@ def add_where_condition(select_query: ast.SelectQuery, condition: ast.Expr) -> N
select_query.where = condition
+def coerce_filter_value(value: object, column_type: str | None) -> object:
+ if value is None or isinstance(value, bool) or not column_type:
+ return value
+
+ while column_type.endswith(")"):
+ for wrapper in ("Nullable(", "LowCardinality("):
+ if column_type.startswith(wrapper):
+ column_type = column_type[len(wrapper) : -1]
+ break
+ else:
+ break
+
+ if column_type == "String" or column_type.startswith("FixedString("):
+ return str(value)
+
+ if not isinstance(value, str):
+ return value
+
+ integer_type = re.fullmatch(r"(U?)Int(8|16|32|64|128|256)", column_type)
+ if integer_type and re.fullmatch(r"[+-]?\d+", value):
+ integer_value = int(value)
+ bits = int(integer_type.group(2))
+ minimum = 0 if integer_type.group(1) else -(2 ** (bits - 1))
+ maximum = 2**bits - 1 if integer_type.group(1) else 2 ** (bits - 1) - 1
+ return integer_value if minimum <= integer_value <= maximum else value
+
+ if column_type in {"Float32", "Float64"}:
+ try:
+ float_value = float(value)
+ if math.isfinite(float_value) and Decimal(value) == Decimal(str(float_value)):
+ return float_value
+ except (InvalidOperation, ValueError, OverflowError):
+ pass
+
+ return value
+
+
def apply_where_filter(
select_query: ast.SelectQuery,
column: str,
- value: str,
+ value: object,
op: ast.CompareOperationOp = ast.CompareOperationOp.Eq,
value_wrapper_fns: list[str] | None = None,
bucket_fn: str | None = None,
+ column_type: str | None = None,
) -> None:
"""Add a comparison filter to WHERE clause.
When bucket_fn is set (range variables), wrap the filter value with the
bucket function so the comparison matches the bucketed column values.
"""
- right_expr: ast.Expr = ast.Constant(value=value)
+ constant_value = value if value_wrapper_fns or bucket_fn else coerce_filter_value(value, column_type)
+ right_expr: ast.Expr = ast.Constant(value=constant_value)
for fn in reversed(value_wrapper_fns or []):
right_expr = ast.Call(name=fn, args=[right_expr])
if bucket_fn:
@@ -541,6 +583,11 @@ class HogQLEndpointStrategy(EndpointQueryStrategy):
for mat_var in self.materialized_variables:
var_value = data.variables.get(mat_var.code_name)
if var_value is not None:
+ column_type = (
+ self.version.saved_query.get_clickhouse_column_type(mat_var.code_name)
+ if self.version.saved_query
+ else None
+ )
apply_where_filter(
select_query,
mat_var.code_name,
@@ -548,6 +595,7 @@ class HogQLEndpointStrategy(EndpointQueryStrategy):
op=mat_var.operator,
value_wrapper_fns=mat_var.value_wrapper_fns,
bucket_fn=mat_var.bucket_fn,
+ column_type=column_type,
)
return None
diff --git a/products/endpoints/backend/tests/test_endpoint_execution.py b/products/endpoints/backend/tests/test_endpoint_execution.py
index 66474c803abc..f690185d776e 100644
--- a/products/endpoints/backend/tests/test_endpoint_execution.py
+++ b/products/endpoints/backend/tests/test_endpoint_execution.py
@@ -619,43 +619,85 @@ class TestEndpointExecution(ClickhouseTestMixin, APIBaseTest):
# MATERIALIZED HOGQL ENDPOINTS
# =========================================================================
- def test_materialized_hogql_endpoint_filters_by_variable(self):
+ @parameterized.expand(
+ [
+ ("number", 424242),
+ ("string", "424242"),
+ ]
+ )
+ def test_materialized_hogql_endpoint_filters_by_variable_type(self, _name: str, capture_id: object):
+ stored_capture_id = "424242"
+ _create_event(
+ event="capture",
+ distinct_id="user1",
+ team=self.team,
+ timestamp="2026-01-05 12:00:00",
+ properties={"capture_id": stored_capture_id},
+ )
+ flush_persons_and_events()
+ capture_id_var = InsightVariable.objects.create(
+ team=self.team,
+ name="Capture ID",
+ code_name="capture_id",
+ type=InsightVariable.Type.STRING,
+ default_value=stored_capture_id,
+ )
+
endpoint = create_endpoint_with_version(
- name="mat_hogql_var",
+ name=f"mat-hogql-var-{_name}",
team=self.team,
query={
"kind": "HogQLQuery",
- "query": "SELECT count() FROM events WHERE event = {variables.event_name}",
+ "query": "SELECT count() FROM events WHERE properties.capture_id = {variables.capture_id}",
"variables": {
- str(self.event_name_var.id): {
- "variableId": str(self.event_name_var.id),
- "code_name": "event_name",
- "value": "$pageview",
+ str(capture_id_var.id): {
+ "variableId": str(capture_id_var.id),
+ "code_name": "capture_id",
+ "value": stored_capture_id,
}
},
},
created_by=self.user,
is_active=True,
)
- self._materialize_endpoint(endpoint)
+ inline_response = self.client.post(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/run/",
+ {"variables": {"capture_id": stored_capture_id}},
+ format="json",
+ )
+ self.assertEqual(inline_response.status_code, status.HTTP_200_OK, inline_response.json())
- # Execute with variable filter - should filter materialized table by event_name
- with mock.patch.object(
- EndpointExecutionService, "_execute_query_and_respond", return_value=Response({})
- ) as mock_exec:
+ saved_query = self._materialize_endpoint(endpoint)
+ saved_query.columns = {
+ "count()": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt64", "valid": True},
+ "capture_id": {"hogql": "StringDatabaseField", "clickhouse": "String", "valid": True},
+ }
+ saved_query.save(update_fields=["columns"])
+
+ def run_query(_team, query, **_kwargs):
+ if (
+ query.modifiers
+ and query.modifiers.useMaterializedViews
+ and stored_capture_id in query.query
+ and f"'{stored_capture_id}'" not in query.query
+ ):
+ raise CHQueryErrorNoCommonType(
+ "There is no supertype for types String, UInt64",
+ code=386,
+ code_name="no_common_type",
+ )
+ return {"results": inline_response.json()["results"], "columns": ["count()"]}
+
+ with mock.patch("products.endpoints.backend.logic.execution.process_query_model", side_effect=run_query):
response = self.client.post(
f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/run/",
- {"variables": {"event_name": "$pageleave"}},
+ {"variables": {"capture_id": capture_id}},
format="json",
)
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- mock_exec.assert_called()
- # Verify the query has WHERE clause filtering by event_name
- query_request_data = mock_exec.call_args[0][0]
- query_sql = query_request_data["query"]["query"].lower()
- self.assertIn("event_name", query_sql)
- self.assertIn("$pageleave", query_sql)
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ self.assertIs(response.json()["endpoint_materialized"], True)
+ self.assertEqual(response.json()["results"], inline_response.json()["results"])
def test_materialized_hogql_endpoint_selects_only_original_columns(self):
endpoint = create_endpoint_with_version(
diff --git a/products/endpoints/backend/tests/test_materialization_analysis.py b/products/endpoints/backend/tests/test_materialization_analysis.py
index 1f8325b77a18..f8c873c34e9b 100644
--- a/products/endpoints/backend/tests/test_materialization_analysis.py
+++ b/products/endpoints/backend/tests/test_materialization_analysis.py
@@ -581,7 +581,9 @@ class TestCombinatorReaggregation(SimpleTestCase):
class TestMaterializedReadPath(SimpleTestCase):
"""Test that the read path applies value_wrapper_fns when filtering the materialized table."""
- def _build_read_query(self, query_str: str, variables_meta: dict, variable_values: dict) -> str:
+ def _build_read_query(
+ self, query_str: str, variables_meta: dict, variable_values: dict, column_type: str | None = None
+ ) -> str:
"""Simulate the materialized read path: analyze variables, then build a SELECT with filters."""
hogql_query = {"kind": "HogQLQuery", "query": query_str, "variables": variables_meta}
_, _, var_infos = analyze_variables_for_materialization(hogql_query)
@@ -601,6 +603,7 @@ class TestMaterializedReadPath(SimpleTestCase):
op=mat_var.operator,
value_wrapper_fns=mat_var.value_wrapper_fns,
bucket_fn=mat_var.bucket_fn,
+ column_type=column_type,
)
return select_query.to_hogql()
@@ -625,14 +628,35 @@ class TestMaterializedReadPath(SimpleTestCase):
assert "toDate('2024-01-15 14:30:00')" in result
- def test_lower_wrapper_applied_to_value(self):
+ @parameterized.expand(
+ [
+ (
+ "lower",
+ "SELECT count() FROM events WHERE lower(event) = lower({variables.value})",
+ "$PageView",
+ None,
+ "lower('$PageView')",
+ ),
+ (
+ "city_hash_with_numeric_column",
+ "SELECT count() FROM events WHERE cityHash64(distinct_id) = cityHash64({variables.value})",
+ "123",
+ "UInt64",
+ "cityHash64('123')",
+ ),
+ ]
+ )
+ def test_wrapper_applied_to_value(
+ self, _name: str, query: str, value: str, column_type: str | None, expected: str
+ ) -> None:
result = self._build_read_query(
- "SELECT count() FROM events WHERE lower(event) = lower({variables.event_name})",
- {"var-1": {"code_name": "event_name", "value": "$PageView"}},
- {"event_name": "$PageView"},
+ query,
+ {"var-1": {"code_name": "value", "value": value}},
+ {"value": value},
+ column_type,
)
- assert "lower('$PageView')" in result
+ assert expected in result
def test_range_with_wrapper_both_sides(self):
result = self._build_read_query(