:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): add typed filter expression engine (#126034)
Mikayla Thompsonc3423e999863
diff --git a/products/signals/backend/filter_expression.py b/products/signals/backend/filter_expression.pynew file mode 100644index 000000000000..436ba02a3738--- /dev/null+++ b/products/signals/backend/filter_expression.py@@ -0,0 +1,156 @@+from collections.abc import Mapping++from posthog.hogql import ast+from posthog.hogql.errors import BaseHogQLError+from posthog.hogql.parser import parse_expr+from posthog.hogql.visitor import clone_expr+++class FilterValidationError(ValueError):+ def __init__(self, code: str, message: str, node: ast.Expr | None = None) -> None:+ super().__init__(message)+ self.code = code+ self.start = node.start if node else None+ self.end = node.end if node else None+++class FilterExpression:+ MAX_LENGTH = 4096+ MAX_NODES = 256+ MAX_DEPTH = 20+ MAX_LIST_ITEMS = 64+ MAX_LITERAL_LENGTH = 1024+ FIELD_TYPES = frozenset({"string", "integer", "array<string>"})++ def __init__(self, expression: str, fields: Mapping[str, str]) -> None:+ self._fields = dict(fields)+ for name, field_type in self._fields.items():+ if field_type not in self.FIELD_TYPES:+ raise FilterValidationError(+ "unsupported_field_type", f"Catalog field {name} has unsupported type {field_type}."+ )+ self._nodes = 0+ self.referenced_fields: set[str] = set()+ self._expression: ast.Expr | None = None+ if len(expression) > self.MAX_LENGTH:+ raise FilterValidationError("expression_limit", "Use an expression of at most 4096 characters.")+ if not expression.strip():+ return+ try:+ parsed = parse_expr(expression)+ except (BaseHogQLError, RecursionError) as error:+ raise FilterValidationError("syntax", "Enter a boolean HogQL expression.") from error+ if self._validate(parsed, 0) != "boolean":+ raise FilterValidationError("boolean_required", "The expression must return a boolean.", parsed)+ self._expression = parsed++ def predicate(self) -> ast.Expr | None:+ return clone_expr(self._expression) if self._expression is not None else None++ def _require(self, valid: bool, node: ast.Expr, message: str) -> None:+ if not valid:+ raise FilterValidationError("type", message, node)++ def _field_type(self, node: ast.Field) -> str:+ if len(node.chain) != 1 or node.chain[0] not in self._fields:+ raise FilterValidationError("unknown_field", "Use an unqualified field from the filter catalog.", node)+ name = str(node.chain[0])+ self.referenced_fields.add(name)+ return self._fields[name]++ def _validate(self, node: ast.Expr, depth: int) -> str:+ self._nodes += 1+ if depth > self.MAX_DEPTH or self._nodes > self.MAX_NODES:+ raise FilterValidationError(+ "complexity_limit", "Simplify the expression's nesting or number of terms.", node+ )+ if isinstance(node, ast.Constant):+ return self._literal_type(node)+ if isinstance(node, ast.Field):+ return self._field_type(node)+ if isinstance(node, ast.Not):+ self._require(self._validate(node.expr, depth + 1) == "boolean", node, "NOT requires a boolean.")+ return "boolean"+ if isinstance(node, ast.And | ast.Or):+ for expr in node.exprs:+ self._require(self._validate(expr, depth + 1) == "boolean", expr, "AND and OR require boolean terms.")+ return "boolean"+ if isinstance(node, ast.Tuple | ast.Array):+ self._require(0 < len(node.exprs) <= self.MAX_LIST_ITEMS, node, "Use between 1 and 64 list items.")+ types = {self._validate(expr, depth + 1) for expr in node.exprs}+ self._require(len(types) == 1 and types <= {"string", "integer"}, node, "Use one scalar type of list item.")+ return f"array<{next(iter(types))}>"+ if isinstance(node, ast.CompareOperation):+ return self._comparison_type(node, depth)+ if isinstance(node, ast.Call):+ return self._call_type(node, depth)+ raise FilterValidationError("unsupported_expression", "This expression form is not supported in filters.", node)++ def _literal_type(self, node: ast.Constant) -> str:+ value = node.value+ if value is None:+ return "null"+ if isinstance(value, bool):+ return "boolean"+ if isinstance(value, int) and -(2**63) <= value < 2**63:+ return "integer"+ if isinstance(value, str) and len(value) <= self.MAX_LITERAL_LENGTH:+ return "string"+ raise FilterValidationError("literal_limit", "Use a bounded string, integer, boolean or null literal.", node)++ def _comparison_type(self, node: ast.CompareOperation, depth: int) -> str:+ if node.op in {ast.CompareOperationOp.In, ast.CompareOperationOp.NotIn} and isinstance(+ node.right, ast.Constant+ ):+ node.right = ast.Tuple(exprs=[node.right], start=node.right.start, end=node.right.end)+ left = self._validate(node.left, depth + 1)+ right = self._validate(node.right, depth + 1)+ if node.op in {ast.CompareOperationOp.In, ast.CompareOperationOp.NotIn}:+ if not isinstance(node.right, ast.Tuple | ast.Array):+ raise FilterValidationError("type", "IN requires a literal list.", node)+ self._require(+ all(isinstance(expr, ast.Constant) for expr in node.right.exprs), node, "IN requires literal values."+ )+ self._require(+ left in {"string", "integer"} and right == f"array<{left}>",+ node,+ "IN values must match the field type.",+ )+ elif node.op in {ast.CompareOperationOp.Eq, ast.CompareOperationOp.NotEq}:+ self._require(+ not left.startswith("array<") and not right.startswith("array<"), node, "Use has to inspect arrays."+ )+ self._require("null" in {left, right} or left == right, node, "Compare values of the same type.")+ elif node.op in {+ ast.CompareOperationOp.Gt,+ ast.CompareOperationOp.GtEq,+ ast.CompareOperationOp.Lt,+ ast.CompareOperationOp.LtEq,+ }:+ self._require(left in {"string", "integer"} and left == right, node, "Use compatible ordered values.")+ else:+ raise FilterValidationError("unsupported_operator", "This comparison operator is not supported.", node)+ return "boolean"++ def _call_type(self, node: ast.Call, depth: int) -> str:+ if node.params is not None or node.distinct or node.within_group or node.order_by or node.filter_expr:+ raise FilterValidationError("unsupported_function", "Function modifiers are not supported.", node)+ types = [self._validate(expr, depth + 1) for expr in node.args]+ name = node.name.lower()+ if name in {"isnull", "isnotnull"} and len(types) == 1:+ node.name = "isNull" if name == "isnull" else "isNotNull"+ return "boolean"+ if name == "not" and types == ["boolean"]:+ node.name = "not"+ return "boolean"+ if (+ name == "has"+ and types == ["array<string>", "string"]+ and isinstance(node.args[0], ast.Field)+ and isinstance(node.args[1], ast.Constant)+ ):+ node.name = "has"+ return "boolean"+ raise FilterValidationError(+ "unsupported_function", "Use a supported function with matching argument types.", node+ )diff --git a/products/signals/backend/filter_recipes.py b/products/signals/backend/filter_recipes.pynew file mode 100644index 000000000000..90cbab800ef9--- /dev/null+++ b/products/signals/backend/filter_recipes.py@@ -0,0 +1,195 @@+from collections.abc import Mapping++from posthog.hogql import ast+from posthog.hogql.database.models import DatabaseField, MapStringDatabaseField+from posthog.hogql.parser import parse_expr+from posthog.hogql.visitor import CloningVisitor, clone_expr++from posthog.dataclasses import frozen++from products.signals.backend.filter_catalog_schema import CatalogBundle, CatalogField, ColumnRecipe, JsonArrayMapRecipe+from products.signals.backend.filter_expression import FilterExpression, FilterValidationError+++def _call(name: str, *args: ast.Expr) -> ast.Call:+ return ast.Call(name=name, args=list(args))+++def _expr(template: str, **values: ast.Expr) -> ast.Expr:+ return parse_expr(template, placeholders=values)+++@frozen+class _JsonValue:+ raw: ast.Expr+ missing: ast.Expr+ invalid: ast.Expr+++@frozen+class FilterProjection:+ field: CatalogField+ value: ast.Expr+ invalid: ast.Expr++ def error(self) -> ast.Expr:+ return _call(+ "throwIf", clone_expr(self.invalid), ast.Constant(value=f"signal_filter:malformed_value:{self.field.name}")+ )++ def select(self) -> ast.Alias:+ failure: ast.Expr = _call("toInt" if self.field.type == "integer" else "toString", self.error())+ if self.field.type == "array<string>":+ failure = ast.Array(exprs=[failure])+ return ast.Alias(+ alias=self.field.name, expr=_call("if", clone_expr(self.invalid), failure, clone_expr(self.value))+ )+++class _RecipeBinder(CloningVisitor):+ def __init__(self, projections: Mapping[str, FilterProjection]) -> None:+ super().__init__()+ self.projections = projections++ def visit_field(self, node: ast.Field) -> ast.Expr:+ return clone_expr(self.projections[str(node.chain[0])].value)+++class WarehouseFilterBindings:+ MAX_JSON_BYTES = 65536+ MAX_ARRAY_ITEMS = 256++ def __init__(self, catalog: CatalogBundle, columns: Mapping[str, DatabaseField], *, source_alias: str) -> None:+ self._fields = {field.name: field.model_copy(deep=True) for field in catalog.fields}+ FilterExpression("", {name: field.type for name, field in self._fields.items()})+ for field in self._fields.values():+ expected = "empty_array" if field.type == "array<string>" else "null"+ if field.missing_value != expected:+ raise FilterValidationError(+ "unsupported_missing_value", f"Catalog field {field.name} requires missing_value={expected}."+ )+ self._columns = dict(columns)+ self._source_alias = source_alias+ self.projections = {name: self._compile(field) for name, field in self._fields.items()}++ def predicate(self, expression: str) -> ast.Expr | None:+ """AND this predicate with scope, time and status restrictions before applying a limit."""+ validated = FilterExpression(expression, {name: field.type for name, field in self._fields.items()})+ predicate = validated.predicate()+ if predicate is None:+ return None+ bound = _call("ifNull", _RecipeBinder(self.projections).visit(predicate), ast.Constant(value=False))+ guards = [+ expr+ for name in sorted(validated.referenced_fields)+ for expr in (clone_expr(self.projections[name].invalid), self.projections[name].error())+ ]+ return _call("multiIf", *guards, bound) if guards else bound++ @staticmethod+ def _physical_type(column: DatabaseField) -> str | None:+ if isinstance(column, MapStringDatabaseField):+ return None+ value = column.get_constant_type()+ if isinstance(value, ast.StringArrayType) or (+ isinstance(value, ast.ArrayType) and isinstance(value.item_type, ast.StringType)+ ):+ return "array<string>"+ if isinstance(value, ast.StringType):+ return "array<string>" if column.array else "string"+ if isinstance(value, ast.IntegerType) and not column.array:+ return "integer"+ return None++ @staticmethod+ def _json_path(raw: ast.Expr, path: list[str]) -> _JsonValue:+ prefixes = [raw]+ for key in path:+ prefixes.append(_call("JSONExtractRaw", prefixes[-1], ast.Constant(value=key)))+ missing = [_expr("{raw} = '' OR toString(JSONType({raw})) = 'Null'", raw=prefix) for prefix in prefixes]+ wrong_shapes = [+ _expr("NOT {missing} AND toString(JSONType({raw})) != 'Object'", missing=missing[index], raw=prefix)+ for index, prefix in enumerate(prefixes[:-1])+ ]+ return _JsonValue(+ raw=prefixes[-1],+ missing=ast.Or(exprs=missing),+ invalid=ast.Or(exprs=[_expr("NOT isValidJSON({raw})", raw=raw), *wrong_shapes]),+ )++ @staticmethod+ def _scalar(raw: ast.Expr, field_type: str) -> _JsonValue:+ allowed = ["String"] if field_type == "string" else ["Int64", "UInt64"]+ wrong_type = _expr(+ "toString(JSONType({raw})) NOT IN {allowed}",+ raw=raw,+ allowed=ast.Tuple(exprs=[ast.Constant(value=item) for item in allowed]),+ )+ value = (+ _call("JSONExtractString", raw)+ if field_type == "string"+ else _call("accurateCastOrNull", _call("JSONExtractRaw", raw), ast.Constant(value="Int64"))+ )+ return _JsonValue(+ raw=value,+ missing=ast.Constant(value=False),+ invalid=_expr("{wrong} OR isNull({value})", wrong=wrong_type, value=value),+ )++ def _compile(self, field: CatalogField) -> FilterProjection:+ recipe = field.binding+ column = self._columns.get(recipe.column)+ if column is None:+ raise FilterValidationError(+ "missing_column", f"Catalog field {field.name} requires column {recipe.column}."+ )+ physical = self._physical_type(column)+ value: ast.Expr = ast.Field(chain=[self._source_alias, recipe.column])+ if isinstance(recipe, ColumnRecipe):+ if physical != field.type:+ raise FilterValidationError("column_type", f"Column {recipe.column} is incompatible with {field.type}.")+ invalid: ast.Expr = ast.Constant(value=False)+ if field.type == "integer":+ converted = _call("accurateCastOrNull", value, ast.Constant(value="Int64"))+ invalid = _expr("NOT isNull({original}) AND isNull({converted})", original=value, converted=converted)+ value = converted+ elif field.type == "array<string>":+ oversized = _expr(+ "length({value}) > {limit}", value=value, limit=ast.Constant(value=self.MAX_ARRAY_ITEMS)+ )+ value = _call("if", oversized, ast.Array(exprs=[]), value)+ invalid = _expr("{oversized} OR arrayExists(x -> isNull(x), {value})", oversized=oversized, value=value)+ return FilterProjection(field=field, value=value, invalid=invalid)+ if physical != "string":+ raise FilterValidationError("column_type", f"JSON recipe {field.name} requires a JSON string column.")+ raw = _call("ifNull", value, ast.Constant(value="null"))+ oversized = _expr("length({raw}) > {limit}", raw=raw, limit=ast.Constant(value=self.MAX_JSON_BYTES))+ bounded = _call("if", oversized, ast.Constant(value="null"), raw)+ path = recipe.array_path if isinstance(recipe, JsonArrayMapRecipe) else recipe.path+ extracted = self._json_path(bounded, path)+ invalid = ast.Or(exprs=[oversized, extracted.invalid])+ if field.type == "array<string>":+ items = _call("JSONExtractArrayRaw", extracted.raw)+ too_many = _expr("length({items}) > {limit}", items=items, limit=ast.Constant(value=self.MAX_ARRAY_ITEMS))+ items = _call("if", too_many, ast.Array(exprs=[]), items)+ item = self._json_path(+ ast.Field(chain=["_filter_item"]), recipe.value_path if isinstance(recipe, JsonArrayMapRecipe) else []+ )+ scalar = self._scalar(item.raw, "string")+ invalid = _expr(+ "{invalid} OR (NOT {missing} AND toString(JSONType({raw})) != 'Array') OR {too_many} OR arrayExists({bad}, {items})",+ invalid=invalid,+ missing=extracted.missing,+ raw=extracted.raw,+ too_many=too_many,+ items=items,+ bad=ast.Lambda(args=["_filter_item"], expr=ast.Or(exprs=[item.missing, item.invalid, scalar.invalid])),+ )+ value = _call("arrayMap", ast.Lambda(args=["_filter_item"], expr=scalar.raw), items)+ else:+ scalar = self._scalar(extracted.raw, field.type)+ invalid = _expr(+ "{invalid} OR (NOT {missing} AND {bad})", invalid=invalid, missing=extracted.missing, bad=scalar.invalid+ )+ value = _call("if", extracted.missing, ast.Constant(value=None), scalar.raw)+ return FilterProjection(field=field, value=value, invalid=invalid)diff --git a/products/signals/backend/test/test_filter_expression.py b/products/signals/backend/test/test_filter_expression.pynew file mode 100644index 000000000000..07c1d655b291--- /dev/null+++ b/products/signals/backend/test/test_filter_expression.py@@ -0,0 +1,113 @@+from django.test import SimpleTestCase++from parameterized import parameterized++from products.signals.backend.filter_expression import FilterExpression, FilterValidationError+++class TestFilterExpression(SimpleTestCase):+ fields = {+ "project_name": "string",+ "priority": "integer",+ "label_names": "array<string>",+ }++ @parameterized.expand(+ [+ "false",+ "true",+ "priority IN (1, 2)",+ "priority IN (1)",+ "project_name NOT IN ('Core')",+ "priority = -1",+ "priority NOT IN (1, 2)",+ "project_name IS NULL",+ "project_name = 'Core' AND NOT has(label_names, 'internal')",+ "isNull(label_names)",+ "HAS(label_names, 'internal')",+ ]+ )+ def test_boolean_contract(self, expression: str) -> None:+ self.assertIsNotNone(FilterExpression(expression, self.fields).predicate())++ @parameterized.expand(+ [+ "SELECT true",+ "(SELECT true)",+ "priority IN (SELECT team_id FROM events)",+ "events.team_id = 1",+ "project_name.foo = 'x'",+ "unknown = 1",+ "{priority} = 1",+ "sum(priority) > 0",+ "count() OVER () > 0",+ "arrayExists(x -> x = 'x', label_names)",+ "sleep(10) = 0",+ "priority + 1 > 2",+ "priority GLOBAL IN (1, 2)",+ "project_name =~ '.*'",+ "priority",+ "1",+ "'true'",+ "NULL",+ "priority AND true",+ "NOT priority",+ "priority = '1'",+ "has(label_names, 1)",+ "has(label_names, project_name)",+ "priority = 1.5",+ "project_name LIKE '%core%'",+ "project_name ILIKE '%core%'",+ "lower(project_name) = 'core'",+ "upper(project_name) = 'CORE'",+ "startsWith(project_name, 'co')",+ "endsWith(project_name, 're')",+ "length(label_names) > 0",+ "empty(label_names)",+ "notEmpty(project_name)",+ "has(priority, 1)",+ "label_names = ['x']",+ "priority IN (project_name, priority)",+ "priority IN (1, '2')",+ "priority IN ('1')",+ "priority IN (priority)",+ "priority IN (NULL)",+ "true; SELECT true",+ "true AS accepted",+ "isNull(priority, priority)",+ "has(('a', 'b'), 'a')",+ "toDateTime('2026-01-02T00:00:00Z') IS NULL",+ ]+ )+ def test_rejects_unsafe_or_untyped_expression(self, expression: str) -> None:+ with self.assertRaises(FilterValidationError):+ FilterExpression(expression, self.fields)++ @parameterized.expand(+ [+ ("length", " " * 4097, "expression_limit"),+ ("depth", "NOT " * 22 + "true", "complexity_limit"),+ ("nodes", " OR ".join(["priority = 1"] * 90), "complexity_limit"),+ ("list", "priority IN (" + ",".join(str(i) for i in range(65)) + ")", "type"),+ ("string", "project_name = '" + "x" * 1025 + "'", "literal_limit"),+ ("integer", "priority = 9223372036854775808", "literal_limit"),+ ]+ )+ def test_complexity_limits(self, _name: str, expression: str, code: str) -> None:+ with self.assertRaises(FilterValidationError) as raised:+ FilterExpression(expression, self.fields)+ self.assertEqual(raised.exception.code, code)++ @parameterized.expand(["boolean", "number", "datetime", "array<integer>", "array<number>", "array<boolean>"])+ def test_rejects_deferred_field_types(self, field_type: str) -> None:+ with self.assertRaises(FilterValidationError) as raised:+ FilterExpression("", {"future_field": field_type})+ self.assertEqual(raised.exception.code, "unsupported_field_type")++ def test_blank_and_structured_errors(self) -> None:+ self.assertIsNone(FilterExpression(" \n ", self.fields).predicate())+ with self.assertRaises(FilterValidationError) as raised:+ FilterExpression("unknown = 1", self.fields)+ self.assertEqual(raised.exception.code, "unknown_field")+ self.assertIsNotNone(raised.exception.start)+ self.assertIsNotNone(raised.exception.end)diff --git a/products/signals/backend/test/test_filter_recipes.py b/products/signals/backend/test/test_filter_recipes.pynew file mode 100644index 000000000000..ecef0a614705--- /dev/null+++ b/products/signals/backend/test/test_filter_recipes.py@@ -0,0 +1,364 @@+import json++from django.test import SimpleTestCase++from parameterized import parameterized++from posthog.hogql import ast+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.database import Database+from posthog.hogql.database.models import (+ DatabaseField,+ IntegerDatabaseField,+ StringArrayDatabaseField,+ StringDatabaseField,+)+from posthog.hogql.printer import prepare_and_print_ast++from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.workload import Workload+from posthog.errors import InternalCHQueryError+from posthog.models import Team++from products.signals.backend.filter_catalog_schema import CatalogBundle, JsonPathRecipe+from products.signals.backend.filter_expression import FilterValidationError+from products.signals.backend.filter_recipes import WarehouseFilterBindings+++def catalog_bundle(field_type: str = "string", kind: str = "json_path", missing: str | None = None) -> CatalogBundle:+ recipe: dict[str, object] = {"kind": kind, "column": "payload"}+ if kind == "json_path":+ recipe["path"] = ["value"]+ elif kind == "json_array_map":+ recipe.update(array_path=["nodes"], value_path=["name"])+ return CatalogBundle.model_validate(+ {+ "schema_version": 1,+ "fields": [+ {+ "name": "value",+ "type": field_type,+ "description": "Synthetic value.",+ "binding": recipe,+ "missing_value": missing+ if missing is not None+ else "empty_array"+ if field_type.startswith("array<")+ else "null",+ "malformed_value": "error",+ }+ ],+ "starter_expressions": [],+ }+ )+++def bindings(+ field_type: str = "string", kind: str = "json_path", missing: str | None = None+) -> WarehouseFilterBindings:+ column: DatabaseField = (+ IntegerDatabaseField(name="payload")+ if kind == "column" and field_type == "integer"+ else StringDatabaseField(name="payload")+ )+ return WarehouseFilterBindings(+ catalog_bundle(field_type, kind, missing), {"payload": column}, source_alias="source"+ )+++class TestFilterRecipeValidation(SimpleTestCase):+ @parameterized.expand(+ ["boolean", "number", "datetime", "array<integer>", "array<number>", "array<boolean>", "array<string>"]+ )+ def test_rejects_unsupported_catalog_configuration(self, field_type: str) -> None:+ with self.assertRaises(FilterValidationError) as raised:+ bindings(field_type, missing="null")+ self.assertEqual(+ raised.exception.code,+ "unsupported_missing_value" if field_type == "array<string>" else "unsupported_field_type",+ )++ def test_rejects_incompatible_columns_before_querying(self) -> None:+ bundle = catalog_bundle()+ with self.assertRaises(FilterValidationError) as raised:+ WarehouseFilterBindings(bundle, {}, source_alias="source")+ self.assertEqual(raised.exception.code, "missing_column")+ with self.assertRaises(FilterValidationError) as raised:+ WarehouseFilterBindings(bundle, {"payload": IntegerDatabaseField(name="payload")}, source_alias="source")+ self.assertEqual(raised.exception.code, "column_type")+ with self.assertRaises(FilterValidationError):+ bindings("array<string>", "column")+++class TestFilterRecipeExecution(SimpleTestCase):+ def assert_malformed(self, engine: WarehouseFilterBindings, payload: str, expression: str | None = None) -> None:+ with self.assertRaisesRegex(InternalCHQueryError, "signal_filter:malformed_value:value") as raised:+ self.execute(engine, payload, expression)+ self.assertEqual(raised.exception.code, 395)++ def execute(+ self, engine: WarehouseFilterBindings, payload: str | int | None, expression: str | None = None+ ) -> object:+ selected: ast.Expr = (+ engine.projections["value"].select()+ if expression is None+ else engine.predicate(expression) or ast.Constant(value=True)+ )+ query = ast.SelectQuery(+ select=[selected],+ select_from=ast.JoinExpr(+ alias="source",+ table=ast.SelectQuery(select=[ast.Alias(alias="payload", expr=ast.Constant(value=payload))]),+ ),+ )+ return self.execute_query(query)[0][0]++ def execute_query(self, query: ast.SelectQuery) -> list[tuple[object, ...]]:+ context = HogQLContext(+ team=Team(id=1, project_id=1),+ database=Database(),+ enable_select_queries=True,+ restricted_properties=set(),+ apply_events_retention_floor=False,+ use_new_events_schema=False,+ )+ sql, _ = prepare_and_print_ast(query, context, "clickhouse")+ return sync_execute(sql, context.values, flush=False, workload=Workload.OFFLINE)++ @parameterized.expand(+ [+ ("string", '{"value":"Core"}', "Core"),+ ("integer", '{"value":0}', 0),+ ("integer", '{"value":-2}', -2),+ ("string", "{}", None),+ ("string", '{"value":null}', None),+ ("string", "null", None),+ ("string", " null ", None),+ ("string", None, None),+ ("string", "{}", 1, "NOT (value = 'Core')"),+ ]+ )+ def test_scalar_values_and_missing(+ self, field_type: str, payload: str | None, expected: object, expression: str | None = None+ ) -> None:+ self.assertEqual(self.execute(bindings(field_type), payload, expression), expected)++ @parameterized.expand(+ [+ ("string", "broken"),+ ("string", ""),+ ("string", "[]"),+ ("string", '{"value":42}'),+ ("string", '{"value":{}}'),+ ("integer", '{"value":"2"}'),+ ("integer", '{"value":1.5}'),+ ("integer", '{"value":9223372036854775808}'),+ ]+ )+ def test_malformed_values_surface_errors(self, field_type: str, payload: str) -> None:+ engine = bindings(field_type)+ self.assert_malformed(engine, payload)+ self.assert_malformed(engine, payload, "true OR isNull(value)")+ self.assert_malformed(engine, payload, "false AND isNull(value)")++ @parameterized.expand(+ [+ ('{"nodes":[{"name":"internal"},{"name":"feature"}]}', ["internal", "feature"]),+ ('{"nodes":[]}', []),+ ('{"nodes":null}', []),+ ("{}", []),+ ("null", []),+ ]+ )+ def test_array_map_and_empty_policy(self, payload: str, expected: object) -> None:+ self.assertEqual(self.execute(bindings("array<string>", "json_array_map", "empty_array"), payload), expected)++ @parameterized.expand(+ [+ "[]",+ '{"nodes":{}}',+ '{"nodes":[{}]}',+ '{"nodes":[{"name":null}]}',+ '{"nodes":[{"name":2}]}',+ '{"nodes":[null]}',+ ]+ )+ def test_array_shape_and_elements_fail_closed(self, payload: str) -> None:+ self.assert_malformed(+ bindings("array<string>", "json_array_map", "empty_array"), payload, "NOT has(value, 'internal')"+ )++ def test_json_array_and_column_recipes(self) -> None:+ self.assertEqual(self.execute(bindings("array<string>"), '{"value":["a","b"]}'), ["a", "b"])+ self.assertEqual(self.execute(bindings("integer", "column"), 0, "value IN (1,2)"), 0)+ self.assertEqual(self.execute(bindings("integer", "column"), 2, "value IN (1,2)"), 1)+ self.assertEqual(self.execute(bindings("integer", "column"), 2, "value IN (2)"), 1)+ self.assertEqual(self.execute(bindings("integer", "column"), 1, "value IN (2)"), 0)+ self.assertEqual(self.execute(bindings("string", "column"), "Core", "value NOT IN ('Core')"), 0)+ self.assertEqual(self.execute(bindings("string", "column"), "Other", "value NOT IN ('Core')"), 1)+ self.assertEqual(self.execute(bindings("string", "column"), "literal' OR true", "value = 'other'"), 0)+ self.assertEqual(self.execute(bindings("integer", "column"), 2, "false"), 0)+ self.assertIsNone(bindings().predicate(""))++ @parameterized.expand(+ [+ (" 2 ", 2),+ ("\t2\t", 2),+ ("\n2\n", 2),+ (" 9223372036854775807 ", 9223372036854775807),+ (" -9223372036854775808 ", -9223372036854775808),+ (" 9223372036854775808 ", None),+ (" -9223372036854775809 ", None),+ ]+ )+ def test_root_json_integer_whitespace(self, payload: str, expected: int | None) -> None:+ bundle = catalog_bundle("integer")+ bundle.fields[0].binding = JsonPathRecipe(kind="json_path", column="payload", path=[])+ engine = WarehouseFilterBindings(+ bundle, {"payload": StringDatabaseField(name="payload")}, source_alias="source"+ )+ if expected is None:+ self.assert_malformed(engine, payload)+ else:+ self.assertEqual(self.execute(engine, payload), expected)+ self.assertEqual(self.execute(engine, payload, f"value = {expected}"), 1)++ def test_runtime_size_bounds(self) -> None:+ for payload, engine in [+ (json.dumps({"value": "x" * 65536}), bindings()),+ (json.dumps({"nodes": [{"name": "a"}] * 257}), bindings("array<string>", "json_array_map")),+ ]:+ self.assert_malformed(engine, payload)++ def test_native_integer_overflow_is_malformed(self) -> None:+ with self.assertRaises(InternalCHQueryError) as raised:+ self.execute(bindings("integer", "column"), 9223372036854775808)+ self.assertEqual(raised.exception.code, 395)++ def test_native_string_array_projection(self) -> None:+ engine = WarehouseFilterBindings(+ catalog_bundle("array<string>", "column"),+ {"payload": StringArrayDatabaseField(name="payload")},+ source_alias="source",+ )+ query = ast.SelectQuery(+ select=[engine.projections["value"].select()],+ select_from=ast.JoinExpr(+ alias="source",+ table=ast.SelectQuery(+ select=[ast.Alias(alias="payload", expr=ast.Array(exprs=[ast.Constant(value="feature")]))]+ ),+ ),+ )+ self.assertEqual(self.execute_query(query), [(["feature"],)])++ def test_shared_column_with_colliding_projection_name(self) -> None:+ bundle = catalog_bundle()+ bundle.fields[0].name = "payload"+ bundle.fields.append(+ bundle.fields[0].model_copy(+ update={"name": "other", "binding": JsonPathRecipe(kind="json_path", column="payload", path=["other"])}+ )+ )+ engine = WarehouseFilterBindings(+ bundle, {"payload": StringDatabaseField(name="payload")}, source_alias="source"+ )+ query = ast.SelectQuery(+ select=[projection.select() for projection in engine.projections.values()],+ select_from=ast.JoinExpr(+ alias="source",+ table=ast.SelectQuery(+ select=[ast.Alias(alias="payload", expr=ast.Constant(value='{"value":"Core","other":"Team"}'))]+ ),+ ),+ where=engine.predicate("payload = 'Core' AND other = 'Team'"),+ )+ self.assertEqual(self.execute_query(query), [("Core", "Team")])++ def test_many_referenced_fields_resolve_and_execute(self) -> None:+ bundle = catalog_bundle(kind="column")+ bundle.fields = [bundle.fields[0].model_copy(update={"name": f"field_{index}"}) for index in range(127)]+ engine = WarehouseFilterBindings(+ bundle, {"payload": StringDatabaseField(name="payload")}, source_alias="source"+ )+ self.assertEqual(+ self.execute(engine, "assigned", " OR ".join(f"isNull(field_{index})" for index in range(127))), 0+ )++ def test_nested_paths_and_literal_path_keys(self) -> None:+ bundle = catalog_bundle()+ bundle.fields[0].binding = JsonPathRecipe(kind="json_path", column="payload", path=["outer", "value"])+ engine = WarehouseFilterBindings(+ bundle, {"payload": StringDatabaseField(name="payload")}, source_alias="source"+ )+ self.assertIsNone(self.execute(engine, "{}"))+ self.assert_malformed(engine, '{"outer":[]}')+ key = "key'); SELECT true; --"+ bundle.fields[0].binding = JsonPathRecipe(kind="json_path", column="payload", path=[key])+ engine = WarehouseFilterBindings(+ bundle, {"payload": StringDatabaseField(name="payload")}, source_alias="source"+ )+ self.assertEqual(self.execute(engine, json.dumps({key: "safe"})), "safe")++ def test_linear_recipes_in_scoped_query(self) -> None:+ specs = [+ ("team_id", "team", "id"),+ ("project_id", "project", "id"),+ ("project_name", "project", "name"),+ ("assignee_id", "assignee", "id"),+ ("state_type", "state", "type"),+ ("label_names", "labels", "name"),+ ]+ fields: list[dict[str, object]] = [+ {+ "name": name,+ "type": "array<string>" if name == "label_names" else "string",+ "description": "Synthetic issue field.",+ "binding": {"kind": "json_array_map", "column": column, "array_path": ["nodes"], "value_path": [key]}+ if name == "label_names"+ else {"kind": "json_path", "column": column, "path": [key]},+ "missing_value": "empty_array" if name == "label_names" else "null",+ "malformed_value": "error",+ }+ for name, column, key in specs+ ]+ fields.append(+ {+ "name": "priority",+ "type": "integer",+ "description": "Synthetic issue priority.",+ "binding": {"kind": "column", "column": "priority"},+ "missing_value": "null",+ "malformed_value": "error",+ }+ )+ bundle = CatalogBundle.model_validate({"schema_version": 1, "fields": fields, "starter_expressions": []})+ payloads: dict[str, str | int] = {+ "team": json.dumps({"id": "team-a", "name": "Engineering", "key": "ENG"}),+ "project": json.dumps({"id": "project-a", "name": "Core"}),+ "assignee": "null",+ "state": json.dumps({"id": "state-a", "name": "Todo", "type": "unstarted", "color": "#000000"}),+ "labels": json.dumps({"nodes": [{"id": "label-a", "name": "feature", "color": "#ffffff"}]}),+ "priority": 2,+ }+ columns: dict[str, DatabaseField] = {name: StringDatabaseField(name=name) for name in payloads}+ columns["priority"] = IntegerDatabaseField(name="priority")+ engine = WarehouseFilterBindings(bundle, columns, source_alias="source")+ predicate = engine.predicate(+ "team_id = 'team-a' AND project_name = 'Core' AND assignee_id IS NULL AND state_type = 'unstarted' AND NOT has(label_names, 'internal') AND priority IN (1,2)"+ )+ assert predicate is not None+ query = ast.SelectQuery(+ select=[projection.select() for projection in engine.projections.values()],+ select_from=ast.JoinExpr(+ alias="source",+ table=ast.SelectQuery(+ select=[ast.Alias(alias=name, expr=ast.Constant(value=value)) for name, value in payloads.items()]+ ),+ ),+ where=ast.And(exprs=[ast.Constant(value=True), predicate]),+ limit=ast.Constant(value=1),+ )+ self.assertEqual(+ self.execute_query(query), [("team-a", "project-a", "Core", None, "unstarted", ["feature"], 2)]+ )