:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): skip JSONDropKeysPool on unrestricted key reads (#121065)
Georgis Andonis66938d6b3d54
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.mdindex 90f981b8119c..63d2e72bb061 100644--- a/posthog/hogql/ACCESS_CONTROL.md+++ b/posthog/hogql/ACCESS_CONTROL.md@@ -186,6 +186,7 @@ They're masked when the query is printed to ClickHouse SQL, so a restricted read- **Explicit reads** (`properties.email`) are replaced with `NULL`, and the resolver refuses to back them with a materialized column — `ClickHousePropertyResolver` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.- **Whole-blob reads** (`SELECT properties` or `SELECT *`) have the restricted keys stripped from the returned JSON via `JSONDropKeysPool(...)` — `ClickHousePrinter._maybe_apply_json_drop_keys()` in `posthog/hogql/printer/clickhouse.py`.+- **Unrestricted key reads** (`properties.$browser`, `JSONExtractString(properties, '$browser')`) read the stored String blob without `JSONDropKeysPool`, because dropping keys cannot change their value. A read keeps the mask when its key overlaps a restricted key by dot segment (`customer` against a restricted `customer.ssn`), or when the key is an integer position — `key_read_overlaps_restriction()` in `posthog/hogql/restricted_properties.py`.Group restrictions retain their group type index, so a same-named property on another group type stays readable. The masking also applies to the Postgres-backed `system.groups.group_properties` field.diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.pyindex dad4e439e57f..bac1a486acdd 100644--- a/posthog/hogql/printer/clickhouse.py+++ b/posthog/hogql/printer/clickhouse.py@@ -1,5 +1,6 @@import re-from collections.abc import Iterable+from collections.abc import Iterable, Iterator+from contextlib import contextmanagerfrom datetime import date, datetimefrom typing import TYPE_CHECKING, ClassVar, Optional, castfrom uuid import UUID@@ -43,6 +44,7 @@ from posthog.hogql.printer.events_json_document import event_document_sql, json_from posthog.hogql.printer.hogql import HogQLPrinterfrom posthog.hogql.restricted_properties import (RESTRICTABLE_JSON_BLOB_COLUMNS,+ key_read_overlaps_restriction,restricted_feature_flag_names,restricted_property_keys_for_table_type,)@@ -143,9 +145,37 @@ ZONED_DATETIME_COERCIBLE_COMPARE_OPS = frozenset()+# ClickHouse JSON functions whose arguments after the document are all keys into it.+_JSON_KEY_PATH_FUNCTIONS = frozenset(+ {+ "JSONHas",+ "JSONLength",+ "JSONArrayLength",+ "JSONType",+ "JSONExtractUInt",+ "JSONExtractInt",+ "JSONExtractFloat",+ "JSONExtractBool",+ "JSONExtractString",+ "JSONExtractKeys",+ "JSONExtractRaw",+ "JSONExtractArrayRaw",+ "JSONExtractKeysAndValuesRaw",+ }+)+# JSON functions that take keys after the document and then a return type as the last argument.+_JSON_FUNCTIONS_WITH_TRAILING_TYPE = frozenset({"JSONExtract", "JSONExtractKeysAndValues"})+++def _group_type_index_of_blob(column_name: str) -> int | None:+ group_column_match = re.fullmatch(r"group(\d+)_properties", column_name)+ return int(group_column_match.group(1)) if group_column_match else None++class ClickHousePrinter(BasePrinter):DIALECT_NAME: ClassVar[HogQLDialect] = "clickhouse"_reads_native_events_table: bool = False+ _unmasked_blob_type: ast.FieldType | None = Nonedef visit_cte(self, node: ast.CTE):if node.materialized is False:@@ -178,6 +208,22 @@ class ClickHousePrinter(BasePrinter):raise QueryError(f"Aggregation '{node.name}' with WITHIN GROUP is not supported in ClickHouse dialect")def _render_function_call(self, node: ast.Call, func_meta) -> str:+ with self._without_json_drop_keys(self._unmasked_json_function_document(node, func_meta)):+ return self._render_clickhouse_function_call(node, func_meta)++ def _unmasked_json_function_document(self, node: ast.Call, func_meta) -> ast.FieldType | None:+ """The blob a constant-key JSON function call may read without JSONDropKeysPool, or None."""+ key_count = len(node.args) - 1+ if func_meta.clickhouse_name in _JSON_FUNCTIONS_WITH_TRAILING_TYPE:+ key_count -= 1+ elif func_meta.clickhouse_name not in _JSON_KEY_PATH_FUNCTIONS:+ return None+ if key_count < 1 or not all(isinstance(arg, ast.Constant) for arg in node.args[1:]):+ return None+ first_key = cast(ast.Constant, node.args[1]).value+ return self._unmasked_key_read_type(node.args[0], first_key)++ def _render_clickhouse_function_call(self, node: ast.Call, func_meta) -> str:args_count = len(node.args) - func_meta.passthrough_suffix_args_countnode_args, passthrough_suffix_args = node.args[:args_count], node.args[args_count:]@@ -683,6 +729,52 @@ class ClickHousePrinter(BasePrinter):finally:self._json_function_argument_depth = depth+ def visit_property_access(self, node: ast.PropertyAccess) -> str:+ first_key = node.keys[0] if node.keys else None+ with self._without_json_drop_keys(self._unmasked_key_read_type(node.expr, first_key)):+ return super().visit_property_access(node)++ def _unmasked_key_read_type(self, document: ast.Expr, first_key: object) -> ast.FieldType | None:+ """The blob that a read of `first_key` may print without JSONDropKeysPool, or None to keep the mask.++ A key that no restricted path overlaps reads the same value from the stored blob as from the masked one, so+ the per-row UDF call only costs time. An integer first key reads by position, which dropping keys shifts.+ Native events documents are rebuilt by the printer and keep their own masking. A user-written alias keeps the+ mask, because ClickHouse lets the rest of the query read the aliased blob.+ """+ if not self.context.restricted_properties or not isinstance(first_key, str):+ return None+ while isinstance(document, ast.Alias) and document.hidden:+ document = document.expr+ if not isinstance(document, ast.Field) or not isinstance(document.type, ast.FieldType):+ return None+ field_type = document.type+ resolved_field = field_type.resolve_database_field(self.context)+ if not isinstance(resolved_field, StringJSONDatabaseField):+ return None+ if resolved_field.name not in RESTRICTABLE_JSON_BLOB_COLUMNS:+ return None+ if self._events_json_document_field(field_type) is not None:+ return None+ keys_to_drop = restricted_property_keys_for_table_type(+ field_type.table_type, self.context, group_type_index=_group_type_index_of_blob(resolved_field.name)+ )+ if key_read_overlaps_restriction(first_key, keys_to_drop):+ return None+ return field_type++ @contextmanager+ def _without_json_drop_keys(self, field_type: ast.FieldType | None) -> Iterator[None]:+ if field_type is None:+ yield+ return+ previous = self._unmasked_blob_type+ self._unmasked_blob_type = field_type+ try:+ yield+ finally:+ self._unmasked_blob_type = previous+def _maybe_apply_json_drop_keys(self, type: ast.FieldType, field_sql: str, mask_feature_flags: bool = False) -> str:"""Wraps a StringJSONDatabaseField in JSONDropKeysPool() to strip restricted property keys@@ -693,7 +785,7 @@ class ClickHousePrinter(BasePrinter):`$feature_flags` map, where JSONDropKeysPool cannot reach a restricted `$feature/<key>`, so the map is droppedand merged back without the restricted flags."""- if not self.context.restricted_properties:+ if not self.context.restricted_properties or type is self._unmasked_blob_type:return field_sqlresolved_field = type.resolve_database_field(self.context)@@ -709,13 +801,8 @@ class ClickHousePrinter(BasePrinter):if resolved_field.name not in RESTRICTABLE_JSON_BLOB_COLUMNS:return field_sql- group_type_index = None- group_column_match = re.fullmatch(r"group(\d+)_properties", resolved_field.name)- if group_column_match:- group_type_index = int(group_column_match.group(1))-keys_to_drop = restricted_property_keys_for_table_type(- type.table_type, self.context, group_type_index=group_type_index+ type.table_type, self.context, group_type_index=_group_type_index_of_blob(resolved_field.name))if not keys_to_drop:return field_sqldiff --git a/posthog/hogql/printer/test/test_property_access_control.py b/posthog/hogql/printer/test/test_property_access_control.pyindex bee685a1525d..935800d0f1bc 100644--- a/posthog/hogql/printer/test/test_property_access_control.py+++ b/posthog/hogql/printer/test/test_property_access_control.py@@ -2,8 +2,6 @@ import uuidfrom posthog.test.base import BaseTest, ClickhouseTestMixin, cleanup_materialized_columns, materialized-from django.conf import settings-from parameterized import parameterizedfrom posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode@@ -149,13 +147,56 @@ class TestRestrictPropertiesInHogQL(BaseTest):assert "NULL AS secret_field" in sqlassert "public_field" in sqlassert "'secret_field'" not in sql- if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA:- assert "JSONDropKeys" not in sql- assert not any("secret_field" in str(v) for v in values.values())- else:- assert "JSONDropKeys" in sql- self._assert_value_present(values, "secret_field")- self._assert_value_present(values, "public_field")+ assert "JSONDropKeys" not in sql+ assert not any("secret_field" in str(v) for v in values.values())++ @parameterized.expand(+ [+ ("unrestricted_key", "properties.public_field", "secret_field", False),+ ("unrestricted_nested_key", "properties.public_field.inner", "secret_field", False),+ (+ "json_function_with_unrestricted_key",+ "JSONExtractString(properties, 'public_field')",+ "secret_field",+ False,+ ),+ ("json_has_unrestricted_key", "JSONHas(properties, 'public_field')", "secret_field", False),+ ("json_function_with_user_alias", "JSONHas(properties AS p, 'public_field') AS h, p", "secret_field", True),+ ("sibling_of_dotted_restriction", "properties.`customer.name`", "customer.ssn", False),+ ("parent_of_dotted_restriction", "properties.customer", "customer.ssn", True),+ ("flat_key_under_restriction", "properties.`secret_field.inner`", "secret_field", True),+ ("json_function_reads_parent", "JSONExtractRaw(properties, 'customer')", "customer.ssn", True),+ ("json_function_reads_by_position", "JSONExtractRaw(properties, 1)", "secret_field", True),+ ("json_function_whole_document", "JSONExtractKeys(properties)", "secret_field", True),+ (+ "json_extract_with_type_only",+ "JSONExtract(properties, 'Tuple(secret_field String)')",+ "secret_field",+ True,+ ),+ ]+ )+ def test_key_read_keeps_json_drop_keys_only_when_it_overlaps_a_restriction(+ self, _name: str, expression: str, restricted_key: str, expect_mask: bool+ ) -> None:+ self.event_prop.name = restricted_key+ self.event_prop.save()+ PropertyAccessControl.objects.create(+ team=self.team,+ property_definition=self.event_prop,+ access_level=PropertyAccessLevel.NONE.value,+ )+ context = HogQLContext(+ team_id=self.team.pk,+ team=self.team,+ user=self.user,+ enable_select_queries=True,+ modifiers=HogQLQueryModifiers(useNewEventsSchema=False),+ )+ sql, _ = prepare_and_print_ast(+ parse_select(f"SELECT {expression} AS x FROM events"), context=context, dialect="clickhouse"+ )+ assert ("JSONDropKeys" in sql) is expect_mask, sqldef test_user_override_allows_access(self):# default: nonediff --git a/posthog/hogql/restricted_properties.py b/posthog/hogql/restricted_properties.pyindex a87214c6701b..825282bbff33 100644--- a/posthog/hogql/restricted_properties.py+++ b/posthog/hogql/restricted_properties.py@@ -149,10 +149,23 @@ def restricted_property_keys_for_table_type(return restricted_keys+def key_read_overlaps_restriction(first_key: str, restricted_keys: Iterable[str]) -> bool:+ """Whether JSONDropKeys can change what a read of the top-level key `first_key` returns.++ JSONDropKeys matches object keys one dot-separated segment at a time. A restricted `a` also drops the flat key+ `a.b`, and a restricted `a.b` also filters inside the value of `a`. Any other key reads the same value from the+ stored blob as from the masked one.+ """+ segments = first_key.split(".")+ for restricted_key in restricted_keys:+ restricted_segments = restricted_key.split(".")+ shared = min(len(segments), len(restricted_segments))+ if segments[:shared] == restricted_segments[:shared]:+ return True+ return False++def native_property_path_overlaps_restriction(property_name: str, table_type: ast.Type, context: HogQLContext) -> bool:if not context.restricted_properties or not context.uses_new_events_schema():return False- return any(- property_name == key or property_name.startswith(key + ".") or key.startswith(property_name + ".")- for key in restricted_property_keys_for_table_type(table_type, context)- )+ return key_read_overlaps_restriction(property_name, restricted_property_keys_for_table_type(table_type, context))diff --git a/posthog/hogql/transforms/test/test_property_types.py b/posthog/hogql/transforms/test/test_property_types.pyindex 2449809b1466..8d34a6108752 100644--- a/posthog/hogql/transforms/test/test_property_types.py+++ b/posthog/hogql/transforms/test/test_property_types.py@@ -227,7 +227,7 @@ class TestNewEventsSchemaArraySubcolumns(_NewEventsSchemaArraySubcolumnsHelpers,{RestrictedProperty(name="$feature/secret", property_type=PropertyDefinition.Type.EVENT)},PropertyGroupsMode.OPTIMIZED,(- f"JSONExtractRaw(if(empty(events.properties), '', {JSON_DROP_KEYS_CLICKHOUSE_NAME}(",+ "JSONExtractRaw(events.properties, ",f"JSONHas(if(empty(events.properties), '', {JSON_DROP_KEYS_CLICKHOUSE_NAME}(",),("properties_group_feature_flags", "mapFilter("),