Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): add typed filter expression engine (#126034)

Mikayla Thompsonc3423e999863
diff --git a/products/signals/backend/filter_expression.py b/products/signals/backend/filter_expression.py
new file mode 100644
index 000000000000..436ba02a3738
--- /dev/null
+++ b/products/signals/backend/filter_expression.py
@@ -0,0 +1,156 @@
+from collections.abc import Mapping
+
+from posthog.hogql import ast
+from posthog.hogql.errors import BaseHogQLError
+from posthog.hogql.parser import parse_expr
+from posthog.hogql.visitor import clone_expr
+
+
+class FilterValidationError(ValueError):
+ def __init__(self, code: str, message: str, node: ast.Expr | None = None) -> None:
+ super().__init__(message)
+ self.code = code
+ self.start = node.start if node else None
+ self.end = node.end if node else None
+
+
+class FilterExpression:
+ MAX_LENGTH = 4096
+ MAX_NODES = 256
+ MAX_DEPTH = 20
+ MAX_LIST_ITEMS = 64
+ MAX_LITERAL_LENGTH = 1024
+ FIELD_TYPES = frozenset({"string", "integer", "array<string>"})
+
+ def __init__(self, expression: str, fields: Mapping[str, str]) -> None:
+ self._fields = dict(fields)
+ for name, field_type in self._fields.items():
+ if field_type not in self.FIELD_TYPES:
+ raise FilterValidationError(
+ "unsupported_field_type", f"Catalog field {name} has unsupported type {field_type}."
+ )
+ self._nodes = 0
+ self.referenced_fields: set[str] = set()
+ self._expression: ast.Expr | None = None
+ if len(expression) > self.MAX_LENGTH:
+ raise FilterValidationError("expression_limit", "Use an expression of at most 4096 characters.")
+ if not expression.strip():
+ return
+ try:
+ parsed = parse_expr(expression)
+ except (BaseHogQLError, RecursionError) as error:
+ raise FilterValidationError("syntax", "Enter a boolean HogQL expression.") from error
+ if self._validate(parsed, 0) != "boolean":
+ raise FilterValidationError("boolean_required", "The expression must return a boolean.", parsed)
+ self._expression = parsed
+
+ def predicate(self) -> ast.Expr | None:
+ return clone_expr(self._expression) if self._expression is not None else None
+
+ def _require(self, valid: bool, node: ast.Expr, message: str) -> None:
+ if not valid:
+ raise FilterValidationError("type", message, node)
+
+ def _field_type(self, node: ast.Field) -> str:
+ if len(node.chain) != 1 or node.chain[0] not in self._fields:
+ raise FilterValidationError("unknown_field", "Use an unqualified field from the filter catalog.", node)
+ name = str(node.chain[0])
+ self.referenced_fields.add(name)
+ return self._fields[name]
+
+ def _validate(self, node: ast.Expr, depth: int) -> str:
+ self._nodes += 1
+ if depth > self.MAX_DEPTH or self._nodes > self.MAX_NODES:
+ raise FilterValidationError(
+ "complexity_limit", "Simplify the expression's nesting or number of terms.", node
+ )
+ if isinstance(node, ast.Constant):
+ return self._literal_type(node)
+ if isinstance(node, ast.Field):
+ return self._field_type(node)
+ if isinstance(node, ast.Not):
+ self._require(self._validate(node.expr, depth + 1) == "boolean", node, "NOT requires a boolean.")
+ return "boolean"
+ if isinstance(node, ast.And | ast.Or):
+ for expr in node.exprs:
+ self._require(self._validate(expr, depth + 1) == "boolean", expr, "AND and OR require boolean terms.")
+ return "boolean"
+ if isinstance(node, ast.Tuple | ast.Array):
+ self._require(0 < len(node.exprs) <= self.MAX_LIST_ITEMS, node, "Use between 1 and 64 list items.")
+ types = {self._validate(expr, depth + 1) for expr in node.exprs}
+ self._require(len(types) == 1 and types <= {"string", "integer"}, node, "Use one scalar type of list item.")
+ return f"array<{next(iter(types))}>"
+ if isinstance(node, ast.CompareOperation):
+ return self._comparison_type(node, depth)
+ if isinstance(node, ast.Call):
+ return self._call_type(node, depth)
+ raise FilterValidationError("unsupported_expression", "This expression form is not supported in filters.", node)
+
+ def _literal_type(self, node: ast.Constant) -> str:
+ value = node.value
+ if value is None:
+ return "null"
+ if isinstance(value, bool):
+ return "boolean"
+ if isinstance(value, int) and -(2**63) <= value < 2**63:
+ return "integer"
+ if isinstance(value, str) and len(value) <= self.MAX_LITERAL_LENGTH:
+ return "string"
+ raise FilterValidationError("literal_limit", "Use a bounded string, integer, boolean or null literal.", node)
+
+ def _comparison_type(self, node: ast.CompareOperation, depth: int) -> str:
+ if node.op in {ast.CompareOperationOp.In, ast.CompareOperationOp.NotIn} and isinstance(
+ node.right, ast.Constant
+ ):
+ node.right = ast.Tuple(exprs=[node.right], start=node.right.start, end=node.right.end)
+ left = self._validate(node.left, depth + 1)
+ right = self._validate(node.right, depth + 1)
+ if node.op in {ast.CompareOperationOp.In, ast.CompareOperationOp.NotIn}:
+ if not isinstance(node.right, ast.Tuple | ast.Array):
+ raise FilterValidationError("type", "IN requires a literal list.", node)
+ self._require(
+ all(isinstance(expr, ast.Constant) for expr in node.right.exprs), node, "IN requires literal values."
+ )
+ self._require(
+ left in {"string", "integer"} and right == f"array<{left}>",
+ node,
+ "IN values must match the field type.",
+ )
+ elif node.op in {ast.CompareOperationOp.Eq, ast.CompareOperationOp.NotEq}:
+ self._require(
+ not left.startswith("array<") and not right.startswith("array<"), node, "Use has to inspect arrays."
+ )
+ self._require("null" in {left, right} or left == right, node, "Compare values of the same type.")
+ elif node.op in {
+ ast.CompareOperationOp.Gt,
+ ast.CompareOperationOp.GtEq,
+ ast.CompareOperationOp.Lt,
+ ast.CompareOperationOp.LtEq,
+ }:
+ self._require(left in {"string", "integer"} and left == right, node, "Use compatible ordered values.")
+ else:
+ raise FilterValidationError("unsupported_operator", "This comparison operator is not supported.", node)
+ return "boolean"
+
+ def _call_type(self, node: ast.Call, depth: int) -> str:
+ if node.params is not None or node.distinct or node.within_group or node.order_by or node.filter_expr:
+ raise FilterValidationError("unsupported_function", "Function modifiers are not supported.", node)
+ types = [self._validate(expr, depth + 1) for expr in node.args]
+ name = node.name.lower()
+ if name in {"isnull", "isnotnull"} and len(types) == 1:
+ node.name = "isNull" if name == "isnull" else "isNotNull"
+ return "boolean"
+ if name == "not" and types == ["boolean"]:
+ node.name = "not"
+ return "boolean"
+ if (
+ name == "has"
+ and types == ["array<string>", "string"]
+ and isinstance(node.args[0], ast.Field)
+ and isinstance(node.args[1], ast.Constant)
+ ):
+ node.name = "has"
+ return "boolean"
+ raise FilterValidationError(
+ "unsupported_function", "Use a supported function with matching argument types.", node
+ )
diff --git a/products/signals/backend/filter_recipes.py b/products/signals/backend/filter_recipes.py
new file mode 100644
index 000000000000..90cbab800ef9
--- /dev/null
+++ b/products/signals/backend/filter_recipes.py
@@ -0,0 +1,195 @@
+from collections.abc import Mapping
+
+from posthog.hogql import ast
+from posthog.hogql.database.models import DatabaseField, MapStringDatabaseField
+from posthog.hogql.parser import parse_expr
+from posthog.hogql.visitor import CloningVisitor, clone_expr
+
+from posthog.dataclasses import frozen
+
+from products.signals.backend.filter_catalog_schema import CatalogBundle, CatalogField, ColumnRecipe, JsonArrayMapRecipe
+from products.signals.backend.filter_expression import FilterExpression, FilterValidationError
+
+
+def _call(name: str, *args: ast.Expr) -> ast.Call:
+ return ast.Call(name=name, args=list(args))
+
+
+def _expr(template: str, **values: ast.Expr) -> ast.Expr:
+ return parse_expr(template, placeholders=values)
+
+
+@frozen
+class _JsonValue:
+ raw: ast.Expr
+ missing: ast.Expr
+ invalid: ast.Expr
+
+
+@frozen
+class FilterProjection:
+ field: CatalogField
+ value: ast.Expr
+ invalid: ast.Expr
+
+ def error(self) -> ast.Expr:
+ return _call(
+ "throwIf", clone_expr(self.invalid), ast.Constant(value=f"signal_filter:malformed_value:{self.field.name}")
+ )
+
+ def select(self) -> ast.Alias:
+ failure: ast.Expr = _call("toInt" if self.field.type == "integer" else "toString", self.error())
+ if self.field.type == "array<string>":
+ failure = ast.Array(exprs=[failure])
+ return ast.Alias(
+ alias=self.field.name, expr=_call("if", clone_expr(self.invalid), failure, clone_expr(self.value))
+ )
+
+
+class _RecipeBinder(CloningVisitor):
+ def __init__(self, projections: Mapping[str, FilterProjection]) -> None:
+ super().__init__()
+ self.projections = projections
+
+ def visit_field(self, node: ast.Field) -> ast.Expr:
+ return clone_expr(self.projections[str(node.chain[0])].value)
+
+
+class WarehouseFilterBindings:
+ MAX_JSON_BYTES = 65536
+ MAX_ARRAY_ITEMS = 256
+
+ def __init__(self, catalog: CatalogBundle, columns: Mapping[str, DatabaseField], *, source_alias: str) -> None:
+ self._fields = {field.name: field.model_copy(deep=True) for field in catalog.fields}
+ FilterExpression("", {name: field.type for name, field in self._fields.items()})
+ for field in self._fields.values():
+ expected = "empty_array" if field.type == "array<string>" else "null"
+ if field.missing_value != expected:
+ raise FilterValidationError(
+ "unsupported_missing_value", f"Catalog field {field.name} requires missing_value={expected}."
+ )
+ self._columns = dict(columns)
+ self._source_alias = source_alias
+ self.projections = {name: self._compile(field) for name, field in self._fields.items()}
+
+ def predicate(self, expression: str) -> ast.Expr | None:
+ """AND this predicate with scope, time and status restrictions before applying a limit."""
+ validated = FilterExpression(expression, {name: field.type for name, field in self._fields.items()})
+ predicate = validated.predicate()
+ if predicate is None:
+ return None
+ bound = _call("ifNull", _RecipeBinder(self.projections).visit(predicate), ast.Constant(value=False))
+ guards = [
+ expr
+ for name in sorted(validated.referenced_fields)
+ for expr in (clone_expr(self.projections[name].invalid), self.projections[name].error())
+ ]
+ return _call("multiIf", *guards, bound) if guards else bound
+
+ @staticmethod
+ def _physical_type(column: DatabaseField) -> str | None:
+ if isinstance(column, MapStringDatabaseField):
+ return None
+ value = column.get_constant_type()
+ if isinstance(value, ast.StringArrayType) or (
+ isinstance(value, ast.ArrayType) and isinstance(value.item_type, ast.StringType)
+ ):
+ return "array<string>"
+ if isinstance(value, ast.StringType):
+ return "array<string>" if column.array else "string"
+ if isinstance(value, ast.IntegerType) and not column.array:
+ return "integer"
+ return None
+
+ @staticmethod
+ def _json_path(raw: ast.Expr, path: list[str]) -> _JsonValue:
+ prefixes = [raw]
+ for key in path:
+ prefixes.append(_call("JSONExtractRaw", prefixes[-1], ast.Constant(value=key)))
+ missing = [_expr("{raw} = '' OR toString(JSONType({raw})) = 'Null'", raw=prefix) for prefix in prefixes]
+ wrong_shapes = [
+ _expr("NOT {missing} AND toString(JSONType({raw})) != 'Object'", missing=missing[index], raw=prefix)
+ for index, prefix in enumerate(prefixes[:-1])
+ ]
+ return _JsonValue(
+ raw=prefixes[-1],
+ missing=ast.Or(exprs=missing),
+ invalid=ast.Or(exprs=[_expr("NOT isValidJSON({raw})", raw=raw), *wrong_shapes]),
+ )
+
+ @staticmethod
+ def _scalar(raw: ast.Expr, field_type: str) -> _JsonValue:
+ allowed = ["String"] if field_type == "string" else ["Int64", "UInt64"]
+ wrong_type = _expr(
+ "toString(JSONType({raw})) NOT IN {allowed}",
+ raw=raw,
+ allowed=ast.Tuple(exprs=[ast.Constant(value=item) for item in allowed]),
+ )
+ value = (
+ _call("JSONExtractString", raw)
+ if field_type == "string"
+ else _call("accurateCastOrNull", _call("JSONExtractRaw", raw), ast.Constant(value="Int64"))
+ )
+ return _JsonValue(
+ raw=value,
+ missing=ast.Constant(value=False),
+ invalid=_expr("{wrong} OR isNull({value})", wrong=wrong_type, value=value),
+ )
+
+ def _compile(self, field: CatalogField) -> FilterProjection:
+ recipe = field.binding
+ column = self._columns.get(recipe.column)
+ if column is None:
+ raise FilterValidationError(
+ "missing_column", f"Catalog field {field.name} requires column {recipe.column}."
+ )
+ physical = self._physical_type(column)
+ value: ast.Expr = ast.Field(chain=[self._source_alias, recipe.column])
+ if isinstance(recipe, ColumnRecipe):
+ if physical != field.type:
+ raise FilterValidationError("column_type", f"Column {recipe.column} is incompatible with {field.type}.")
+ invalid: ast.Expr = ast.Constant(value=False)
+ if field.type == "integer":
+ converted = _call("accurateCastOrNull", value, ast.Constant(value="Int64"))
+ invalid = _expr("NOT isNull({original}) AND isNull({converted})", original=value, converted=converted)
+ value = converted
+ elif field.type == "array<string>":
+ oversized = _expr(
+ "length({value}) > {limit}", value=value, limit=ast.Constant(value=self.MAX_ARRAY_ITEMS)
+ )
+ value = _call("if", oversized, ast.Array(exprs=[]), value)
+ invalid = _expr("{oversized} OR arrayExists(x -> isNull(x), {value})", oversized=oversized, value=value)
+ return FilterProjection(field=field, value=value, invalid=invalid)
+ if physical != "string":
+ raise FilterValidationError("column_type", f"JSON recipe {field.name} requires a JSON string column.")
+ raw = _call("ifNull", value, ast.Constant(value="null"))
+ oversized = _expr("length({raw}) > {limit}", raw=raw, limit=ast.Constant(value=self.MAX_JSON_BYTES))
+ bounded = _call("if", oversized, ast.Constant(value="null"), raw)
+ path = recipe.array_path if isinstance(recipe, JsonArrayMapRecipe) else recipe.path
+ extracted = self._json_path(bounded, path)
+ invalid = ast.Or(exprs=[oversized, extracted.invalid])
+ if field.type == "array<string>":
+ items = _call("JSONExtractArrayRaw", extracted.raw)
+ too_many = _expr("length({items}) > {limit}", items=items, limit=ast.Constant(value=self.MAX_ARRAY_ITEMS))
+ items = _call("if", too_many, ast.Array(exprs=[]), items)
+ item = self._json_path(
+ ast.Field(chain=["_filter_item"]), recipe.value_path if isinstance(recipe, JsonArrayMapRecipe) else []
+ )
+ scalar = self._scalar(item.raw, "string")
+ invalid = _expr(
+ "{invalid} OR (NOT {missing} AND toString(JSONType({raw})) != 'Array') OR {too_many} OR arrayExists({bad}, {items})",
+ invalid=invalid,
+ missing=extracted.missing,
+ raw=extracted.raw,
+ too_many=too_many,
+ items=items,
+ bad=ast.Lambda(args=["_filter_item"], expr=ast.Or(exprs=[item.missing, item.invalid, scalar.invalid])),
+ )
+ value = _call("arrayMap", ast.Lambda(args=["_filter_item"], expr=scalar.raw), items)
+ else:
+ scalar = self._scalar(extracted.raw, field.type)
+ invalid = _expr(
+ "{invalid} OR (NOT {missing} AND {bad})", invalid=invalid, missing=extracted.missing, bad=scalar.invalid
+ )
+ value = _call("if", extracted.missing, ast.Constant(value=None), scalar.raw)
+ return FilterProjection(field=field, value=value, invalid=invalid)
diff --git a/products/signals/backend/test/test_filter_expression.py b/products/signals/backend/test/test_filter_expression.py
new file mode 100644
index 000000000000..07c1d655b291
--- /dev/null
+++ b/products/signals/backend/test/test_filter_expression.py
@@ -0,0 +1,113 @@
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from products.signals.backend.filter_expression import FilterExpression, FilterValidationError
+
+
+class TestFilterExpression(SimpleTestCase):
+ fields = {
+ "project_name": "string",
+ "priority": "integer",
+ "label_names": "array<string>",
+ }
+
+ @parameterized.expand(
+ [
+ "false",
+ "true",
+ "priority IN (1, 2)",
+ "priority IN (1)",
+ "project_name NOT IN ('Core')",
+ "priority = -1",
+ "priority NOT IN (1, 2)",
+ "project_name IS NULL",
+ "project_name = 'Core' AND NOT has(label_names, 'internal')",
+ "isNull(label_names)",
+ "HAS(label_names, 'internal')",
+ ]
+ )
+ def test_boolean_contract(self, expression: str) -> None:
+ self.assertIsNotNone(FilterExpression(expression, self.fields).predicate())
+
+ @parameterized.expand(
+ [
+ "SELECT true",
+ "(SELECT true)",
+ "priority IN (SELECT team_id FROM events)",
+ "events.team_id = 1",
+ "project_name.foo = 'x'",
+ "unknown = 1",
+ "{priority} = 1",
+ "sum(priority) > 0",
+ "count() OVER () > 0",
+ "arrayExists(x -> x = 'x', label_names)",
+ "sleep(10) = 0",
+ "priority + 1 > 2",
+ "priority GLOBAL IN (1, 2)",
+ "project_name =~ '.*'",
+ "priority",
+ "1",
+ "'true'",
+ "NULL",
+ "priority AND true",
+ "NOT priority",
+ "priority = '1'",
+ "has(label_names, 1)",
+ "has(label_names, project_name)",
+ "priority = 1.5",
+ "project_name LIKE '%core%'",
+ "project_name ILIKE '%core%'",
+ "lower(project_name) = 'core'",
+ "upper(project_name) = 'CORE'",
+ "startsWith(project_name, 'co')",
+ "endsWith(project_name, 're')",
+ "length(label_names) > 0",
+ "empty(label_names)",
+ "notEmpty(project_name)",
+ "has(priority, 1)",
+ "label_names = ['x']",
+ "priority IN (project_name, priority)",
+ "priority IN (1, '2')",
+ "priority IN ('1')",
+ "priority IN (priority)",
+ "priority IN (NULL)",
+ "true; SELECT true",
+ "true AS accepted",
+ "isNull(priority, priority)",
+ "has(('a', 'b'), 'a')",
+ "toDateTime('2026-01-02T00:00:00Z') IS NULL",
+ ]
+ )
+ def test_rejects_unsafe_or_untyped_expression(self, expression: str) -> None:
+ with self.assertRaises(FilterValidationError):
+ FilterExpression(expression, self.fields)
+
+ @parameterized.expand(
+ [
+ ("length", " " * 4097, "expression_limit"),
+ ("depth", "NOT " * 22 + "true", "complexity_limit"),
+ ("nodes", " OR ".join(["priority = 1"] * 90), "complexity_limit"),
+ ("list", "priority IN (" + ",".join(str(i) for i in range(65)) + ")", "type"),
+ ("string", "project_name = '" + "x" * 1025 + "'", "literal_limit"),
+ ("integer", "priority = 9223372036854775808", "literal_limit"),
+ ]
+ )
+ def test_complexity_limits(self, _name: str, expression: str, code: str) -> None:
+ with self.assertRaises(FilterValidationError) as raised:
+ FilterExpression(expression, self.fields)
+ self.assertEqual(raised.exception.code, code)
+
+ @parameterized.expand(["boolean", "number", "datetime", "array<integer>", "array<number>", "array<boolean>"])
+ def test_rejects_deferred_field_types(self, field_type: str) -> None:
+ with self.assertRaises(FilterValidationError) as raised:
+ FilterExpression("", {"future_field": field_type})
+ self.assertEqual(raised.exception.code, "unsupported_field_type")
+
+ def test_blank_and_structured_errors(self) -> None:
+ self.assertIsNone(FilterExpression(" \n ", self.fields).predicate())
+ with self.assertRaises(FilterValidationError) as raised:
+ FilterExpression("unknown = 1", self.fields)
+ self.assertEqual(raised.exception.code, "unknown_field")
+ self.assertIsNotNone(raised.exception.start)
+ self.assertIsNotNone(raised.exception.end)
diff --git a/products/signals/backend/test/test_filter_recipes.py b/products/signals/backend/test/test_filter_recipes.py
new file mode 100644
index 000000000000..ecef0a614705
--- /dev/null
+++ b/products/signals/backend/test/test_filter_recipes.py
@@ -0,0 +1,364 @@
+import json
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.database.models import (
+ DatabaseField,
+ IntegerDatabaseField,
+ StringArrayDatabaseField,
+ StringDatabaseField,
+)
+from posthog.hogql.printer import prepare_and_print_ast
+
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.workload import Workload
+from posthog.errors import InternalCHQueryError
+from posthog.models import Team
+
+from products.signals.backend.filter_catalog_schema import CatalogBundle, JsonPathRecipe
+from products.signals.backend.filter_expression import FilterValidationError
+from products.signals.backend.filter_recipes import WarehouseFilterBindings
+
+
+def catalog_bundle(field_type: str = "string", kind: str = "json_path", missing: str | None = None) -> CatalogBundle:
+ recipe: dict[str, object] = {"kind": kind, "column": "payload"}
+ if kind == "json_path":
+ recipe["path"] = ["value"]
+ elif kind == "json_array_map":
+ recipe.update(array_path=["nodes"], value_path=["name"])
+ return CatalogBundle.model_validate(
+ {
+ "schema_version": 1,
+ "fields": [
+ {
+ "name": "value",
+ "type": field_type,
+ "description": "Synthetic value.",
+ "binding": recipe,
+ "missing_value": missing
+ if missing is not None
+ else "empty_array"
+ if field_type.startswith("array<")
+ else "null",
+ "malformed_value": "error",
+ }
+ ],
+ "starter_expressions": [],
+ }
+ )
+
+
+def bindings(
+ field_type: str = "string", kind: str = "json_path", missing: str | None = None
+) -> WarehouseFilterBindings:
+ column: DatabaseField = (
+ IntegerDatabaseField(name="payload")
+ if kind == "column" and field_type == "integer"
+ else StringDatabaseField(name="payload")
+ )
+ return WarehouseFilterBindings(
+ catalog_bundle(field_type, kind, missing), {"payload": column}, source_alias="source"
+ )
+
+
+class TestFilterRecipeValidation(SimpleTestCase):
+ @parameterized.expand(
+ ["boolean", "number", "datetime", "array<integer>", "array<number>", "array<boolean>", "array<string>"]
+ )
+ def test_rejects_unsupported_catalog_configuration(self, field_type: str) -> None:
+ with self.assertRaises(FilterValidationError) as raised:
+ bindings(field_type, missing="null")
+ self.assertEqual(
+ raised.exception.code,
+ "unsupported_missing_value" if field_type == "array<string>" else "unsupported_field_type",
+ )
+
+ def test_rejects_incompatible_columns_before_querying(self) -> None:
+ bundle = catalog_bundle()
+ with self.assertRaises(FilterValidationError) as raised:
+ WarehouseFilterBindings(bundle, {}, source_alias="source")
+ self.assertEqual(raised.exception.code, "missing_column")
+ with self.assertRaises(FilterValidationError) as raised:
+ WarehouseFilterBindings(bundle, {"payload": IntegerDatabaseField(name="payload")}, source_alias="source")
+ self.assertEqual(raised.exception.code, "column_type")
+ with self.assertRaises(FilterValidationError):
+ bindings("array<string>", "column")
+
+
+class TestFilterRecipeExecution(SimpleTestCase):
+ def assert_malformed(self, engine: WarehouseFilterBindings, payload: str, expression: str | None = None) -> None:
+ with self.assertRaisesRegex(InternalCHQueryError, "signal_filter:malformed_value:value") as raised:
+ self.execute(engine, payload, expression)
+ self.assertEqual(raised.exception.code, 395)
+
+ def execute(
+ self, engine: WarehouseFilterBindings, payload: str | int | None, expression: str | None = None
+ ) -> object:
+ selected: ast.Expr = (
+ engine.projections["value"].select()
+ if expression is None
+ else engine.predicate(expression) or ast.Constant(value=True)
+ )
+ query = ast.SelectQuery(
+ select=[selected],
+ select_from=ast.JoinExpr(
+ alias="source",
+ table=ast.SelectQuery(select=[ast.Alias(alias="payload", expr=ast.Constant(value=payload))]),
+ ),
+ )
+ return self.execute_query(query)[0][0]
+
+ def execute_query(self, query: ast.SelectQuery) -> list[tuple[object, ...]]:
+ context = HogQLContext(
+ team=Team(id=1, project_id=1),
+ database=Database(),
+ enable_select_queries=True,
+ restricted_properties=set(),
+ apply_events_retention_floor=False,
+ use_new_events_schema=False,
+ )
+ sql, _ = prepare_and_print_ast(query, context, "clickhouse")
+ return sync_execute(sql, context.values, flush=False, workload=Workload.OFFLINE)
+
+ @parameterized.expand(
+ [
+ ("string", '{"value":"Core"}', "Core"),
+ ("integer", '{"value":0}', 0),
+ ("integer", '{"value":-2}', -2),
+ ("string", "{}", None),
+ ("string", '{"value":null}', None),
+ ("string", "null", None),
+ ("string", " null ", None),
+ ("string", None, None),
+ ("string", "{}", 1, "NOT (value = 'Core')"),
+ ]
+ )
+ def test_scalar_values_and_missing(
+ self, field_type: str, payload: str | None, expected: object, expression: str | None = None
+ ) -> None:
+ self.assertEqual(self.execute(bindings(field_type), payload, expression), expected)
+
+ @parameterized.expand(
+ [
+ ("string", "broken"),
+ ("string", ""),
+ ("string", "[]"),
+ ("string", '{"value":42}'),
+ ("string", '{"value":{}}'),
+ ("integer", '{"value":"2"}'),
+ ("integer", '{"value":1.5}'),
+ ("integer", '{"value":9223372036854775808}'),
+ ]
+ )
+ def test_malformed_values_surface_errors(self, field_type: str, payload: str) -> None:
+ engine = bindings(field_type)
+ self.assert_malformed(engine, payload)
+ self.assert_malformed(engine, payload, "true OR isNull(value)")
+ self.assert_malformed(engine, payload, "false AND isNull(value)")
+
+ @parameterized.expand(
+ [
+ ('{"nodes":[{"name":"internal"},{"name":"feature"}]}', ["internal", "feature"]),
+ ('{"nodes":[]}', []),
+ ('{"nodes":null}', []),
+ ("{}", []),
+ ("null", []),
+ ]
+ )
+ def test_array_map_and_empty_policy(self, payload: str, expected: object) -> None:
+ self.assertEqual(self.execute(bindings("array<string>", "json_array_map", "empty_array"), payload), expected)
+
+ @parameterized.expand(
+ [
+ "[]",
+ '{"nodes":{}}',
+ '{"nodes":[{}]}',
+ '{"nodes":[{"name":null}]}',
+ '{"nodes":[{"name":2}]}',
+ '{"nodes":[null]}',
+ ]
+ )
+ def test_array_shape_and_elements_fail_closed(self, payload: str) -> None:
+ self.assert_malformed(
+ bindings("array<string>", "json_array_map", "empty_array"), payload, "NOT has(value, 'internal')"
+ )
+
+ def test_json_array_and_column_recipes(self) -> None:
+ self.assertEqual(self.execute(bindings("array<string>"), '{"value":["a","b"]}'), ["a", "b"])
+ self.assertEqual(self.execute(bindings("integer", "column"), 0, "value IN (1,2)"), 0)
+ self.assertEqual(self.execute(bindings("integer", "column"), 2, "value IN (1,2)"), 1)
+ self.assertEqual(self.execute(bindings("integer", "column"), 2, "value IN (2)"), 1)
+ self.assertEqual(self.execute(bindings("integer", "column"), 1, "value IN (2)"), 0)
+ self.assertEqual(self.execute(bindings("string", "column"), "Core", "value NOT IN ('Core')"), 0)
+ self.assertEqual(self.execute(bindings("string", "column"), "Other", "value NOT IN ('Core')"), 1)
+ self.assertEqual(self.execute(bindings("string", "column"), "literal' OR true", "value = 'other'"), 0)
+ self.assertEqual(self.execute(bindings("integer", "column"), 2, "false"), 0)
+ self.assertIsNone(bindings().predicate(""))
+
+ @parameterized.expand(
+ [
+ (" 2 ", 2),
+ ("\t2\t", 2),
+ ("\n2\n", 2),
+ (" 9223372036854775807 ", 9223372036854775807),
+ (" -9223372036854775808 ", -9223372036854775808),
+ (" 9223372036854775808 ", None),
+ (" -9223372036854775809 ", None),
+ ]
+ )
+ def test_root_json_integer_whitespace(self, payload: str, expected: int | None) -> None:
+ bundle = catalog_bundle("integer")
+ bundle.fields[0].binding = JsonPathRecipe(kind="json_path", column="payload", path=[])
+ engine = WarehouseFilterBindings(
+ bundle, {"payload": StringDatabaseField(name="payload")}, source_alias="source"
+ )
+ if expected is None:
+ self.assert_malformed(engine, payload)
+ else:
+ self.assertEqual(self.execute(engine, payload), expected)
+ self.assertEqual(self.execute(engine, payload, f"value = {expected}"), 1)
+
+ def test_runtime_size_bounds(self) -> None:
+ for payload, engine in [
+ (json.dumps({"value": "x" * 65536}), bindings()),
+ (json.dumps({"nodes": [{"name": "a"}] * 257}), bindings("array<string>", "json_array_map")),
+ ]:
+ self.assert_malformed(engine, payload)
+
+ def test_native_integer_overflow_is_malformed(self) -> None:
+ with self.assertRaises(InternalCHQueryError) as raised:
+ self.execute(bindings("integer", "column"), 9223372036854775808)
+ self.assertEqual(raised.exception.code, 395)
+
+ def test_native_string_array_projection(self) -> None:
+ engine = WarehouseFilterBindings(
+ catalog_bundle("array<string>", "column"),
+ {"payload": StringArrayDatabaseField(name="payload")},
+ source_alias="source",
+ )
+ query = ast.SelectQuery(
+ select=[engine.projections["value"].select()],
+ select_from=ast.JoinExpr(
+ alias="source",
+ table=ast.SelectQuery(
+ select=[ast.Alias(alias="payload", expr=ast.Array(exprs=[ast.Constant(value="feature")]))]
+ ),
+ ),
+ )
+ self.assertEqual(self.execute_query(query), [(["feature"],)])
+
+ def test_shared_column_with_colliding_projection_name(self) -> None:
+ bundle = catalog_bundle()
+ bundle.fields[0].name = "payload"
+ bundle.fields.append(
+ bundle.fields[0].model_copy(
+ update={"name": "other", "binding": JsonPathRecipe(kind="json_path", column="payload", path=["other"])}
+ )
+ )
+ engine = WarehouseFilterBindings(
+ bundle, {"payload": StringDatabaseField(name="payload")}, source_alias="source"
+ )
+ query = ast.SelectQuery(
+ select=[projection.select() for projection in engine.projections.values()],
+ select_from=ast.JoinExpr(
+ alias="source",
+ table=ast.SelectQuery(
+ select=[ast.Alias(alias="payload", expr=ast.Constant(value='{"value":"Core","other":"Team"}'))]
+ ),
+ ),
+ where=engine.predicate("payload = 'Core' AND other = 'Team'"),
+ )
+ self.assertEqual(self.execute_query(query), [("Core", "Team")])
+
+ def test_many_referenced_fields_resolve_and_execute(self) -> None:
+ bundle = catalog_bundle(kind="column")
+ bundle.fields = [bundle.fields[0].model_copy(update={"name": f"field_{index}"}) for index in range(127)]
+ engine = WarehouseFilterBindings(
+ bundle, {"payload": StringDatabaseField(name="payload")}, source_alias="source"
+ )
+ self.assertEqual(
+ self.execute(engine, "assigned", " OR ".join(f"isNull(field_{index})" for index in range(127))), 0
+ )
+
+ def test_nested_paths_and_literal_path_keys(self) -> None:
+ bundle = catalog_bundle()
+ bundle.fields[0].binding = JsonPathRecipe(kind="json_path", column="payload", path=["outer", "value"])
+ engine = WarehouseFilterBindings(
+ bundle, {"payload": StringDatabaseField(name="payload")}, source_alias="source"
+ )
+ self.assertIsNone(self.execute(engine, "{}"))
+ self.assert_malformed(engine, '{"outer":[]}')
+ key = "key'); SELECT true; --"
+ bundle.fields[0].binding = JsonPathRecipe(kind="json_path", column="payload", path=[key])
+ engine = WarehouseFilterBindings(
+ bundle, {"payload": StringDatabaseField(name="payload")}, source_alias="source"
+ )
+ self.assertEqual(self.execute(engine, json.dumps({key: "safe"})), "safe")
+
+ def test_linear_recipes_in_scoped_query(self) -> None:
+ specs = [
+ ("team_id", "team", "id"),
+ ("project_id", "project", "id"),
+ ("project_name", "project", "name"),
+ ("assignee_id", "assignee", "id"),
+ ("state_type", "state", "type"),
+ ("label_names", "labels", "name"),
+ ]
+ fields: list[dict[str, object]] = [
+ {
+ "name": name,
+ "type": "array<string>" if name == "label_names" else "string",
+ "description": "Synthetic issue field.",
+ "binding": {"kind": "json_array_map", "column": column, "array_path": ["nodes"], "value_path": [key]}
+ if name == "label_names"
+ else {"kind": "json_path", "column": column, "path": [key]},
+ "missing_value": "empty_array" if name == "label_names" else "null",
+ "malformed_value": "error",
+ }
+ for name, column, key in specs
+ ]
+ fields.append(
+ {
+ "name": "priority",
+ "type": "integer",
+ "description": "Synthetic issue priority.",
+ "binding": {"kind": "column", "column": "priority"},
+ "missing_value": "null",
+ "malformed_value": "error",
+ }
+ )
+ bundle = CatalogBundle.model_validate({"schema_version": 1, "fields": fields, "starter_expressions": []})
+ payloads: dict[str, str | int] = {
+ "team": json.dumps({"id": "team-a", "name": "Engineering", "key": "ENG"}),
+ "project": json.dumps({"id": "project-a", "name": "Core"}),
+ "assignee": "null",
+ "state": json.dumps({"id": "state-a", "name": "Todo", "type": "unstarted", "color": "#000000"}),
+ "labels": json.dumps({"nodes": [{"id": "label-a", "name": "feature", "color": "#ffffff"}]}),
+ "priority": 2,
+ }
+ columns: dict[str, DatabaseField] = {name: StringDatabaseField(name=name) for name in payloads}
+ columns["priority"] = IntegerDatabaseField(name="priority")
+ engine = WarehouseFilterBindings(bundle, columns, source_alias="source")
+ predicate = engine.predicate(
+ "team_id = 'team-a' AND project_name = 'Core' AND assignee_id IS NULL AND state_type = 'unstarted' AND NOT has(label_names, 'internal') AND priority IN (1,2)"
+ )
+ assert predicate is not None
+ query = ast.SelectQuery(
+ select=[projection.select() for projection in engine.projections.values()],
+ select_from=ast.JoinExpr(
+ alias="source",
+ table=ast.SelectQuery(
+ select=[ast.Alias(alias=name, expr=ast.Constant(value=value)) for name, value in payloads.items()]
+ ),
+ ),
+ where=ast.And(exprs=[ast.Constant(value=True), predicate]),
+ limit=ast.Constant(value=1),
+ )
+ self.assertEqual(
+ self.execute_query(query), [("team-a", "project-a", "Core", None, "unstarted", ["feature"], 2)]
+ )