Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): skip JSONDropKeysPool on unrestricted key reads (#121065)

Georgis Andonis66938d6b3d54
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.md
index 90f981b8119c..63d2e72bb061 100644
--- a/posthog/hogql/ACCESS_CONTROL.md
+++ b/posthog/hogql/ACCESS_CONTROL.md
@@ -186,6 +186,7 @@ They're masked when the query is printed to ClickHouse SQL, so a restricted read
- **Explicit reads** (`properties.email`) are replaced with `NULL`, and the resolver refuses to back them with a materialized column — `ClickHousePropertyResolver` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.
- **Whole-blob reads** (`SELECT properties` or `SELECT *`) have the restricted keys stripped from the returned JSON via `JSONDropKeysPool(...)` — `ClickHousePrinter._maybe_apply_json_drop_keys()` in `posthog/hogql/printer/clickhouse.py`.
+- **Unrestricted key reads** (`properties.$browser`, `JSONExtractString(properties, '$browser')`) read the stored String blob without `JSONDropKeysPool`, because dropping keys cannot change their value. A read keeps the mask when its key overlaps a restricted key by dot segment (`customer` against a restricted `customer.ssn`), or when the key is an integer position — `key_read_overlaps_restriction()` in `posthog/hogql/restricted_properties.py`.
Group restrictions retain their group type index, so a same-named property on another group type stays readable. The masking also applies to the Postgres-backed `system.groups.group_properties` field.
diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.py
index dad4e439e57f..bac1a486acdd 100644
--- a/posthog/hogql/printer/clickhouse.py
+++ b/posthog/hogql/printer/clickhouse.py
@@ -1,5 +1,6 @@
import re
-from collections.abc import Iterable
+from collections.abc import Iterable, Iterator
+from contextlib import contextmanager
from datetime import date, datetime
from typing import TYPE_CHECKING, ClassVar, Optional, cast
from uuid import UUID
@@ -43,6 +44,7 @@ from posthog.hogql.printer.events_json_document import event_document_sql, json_
from posthog.hogql.printer.hogql import HogQLPrinter
from posthog.hogql.restricted_properties import (
RESTRICTABLE_JSON_BLOB_COLUMNS,
+ key_read_overlaps_restriction,
restricted_feature_flag_names,
restricted_property_keys_for_table_type,
)
@@ -143,9 +145,37 @@ ZONED_DATETIME_COERCIBLE_COMPARE_OPS = frozenset(
)
+# ClickHouse JSON functions whose arguments after the document are all keys into it.
+_JSON_KEY_PATH_FUNCTIONS = frozenset(
+ {
+ "JSONHas",
+ "JSONLength",
+ "JSONArrayLength",
+ "JSONType",
+ "JSONExtractUInt",
+ "JSONExtractInt",
+ "JSONExtractFloat",
+ "JSONExtractBool",
+ "JSONExtractString",
+ "JSONExtractKeys",
+ "JSONExtractRaw",
+ "JSONExtractArrayRaw",
+ "JSONExtractKeysAndValuesRaw",
+ }
+)
+# JSON functions that take keys after the document and then a return type as the last argument.
+_JSON_FUNCTIONS_WITH_TRAILING_TYPE = frozenset({"JSONExtract", "JSONExtractKeysAndValues"})
+
+
+def _group_type_index_of_blob(column_name: str) -> int | None:
+ group_column_match = re.fullmatch(r"group(\d+)_properties", column_name)
+ return int(group_column_match.group(1)) if group_column_match else None
+
+
class ClickHousePrinter(BasePrinter):
DIALECT_NAME: ClassVar[HogQLDialect] = "clickhouse"
_reads_native_events_table: bool = False
+ _unmasked_blob_type: ast.FieldType | None = None
def visit_cte(self, node: ast.CTE):
if node.materialized is False:
@@ -178,6 +208,22 @@ class ClickHousePrinter(BasePrinter):
raise QueryError(f"Aggregation '{node.name}' with WITHIN GROUP is not supported in ClickHouse dialect")
def _render_function_call(self, node: ast.Call, func_meta) -> str:
+ with self._without_json_drop_keys(self._unmasked_json_function_document(node, func_meta)):
+ return self._render_clickhouse_function_call(node, func_meta)
+
+ def _unmasked_json_function_document(self, node: ast.Call, func_meta) -> ast.FieldType | None:
+ """The blob a constant-key JSON function call may read without JSONDropKeysPool, or None."""
+ key_count = len(node.args) - 1
+ if func_meta.clickhouse_name in _JSON_FUNCTIONS_WITH_TRAILING_TYPE:
+ key_count -= 1
+ elif func_meta.clickhouse_name not in _JSON_KEY_PATH_FUNCTIONS:
+ return None
+ if key_count < 1 or not all(isinstance(arg, ast.Constant) for arg in node.args[1:]):
+ return None
+ first_key = cast(ast.Constant, node.args[1]).value
+ return self._unmasked_key_read_type(node.args[0], first_key)
+
+ def _render_clickhouse_function_call(self, node: ast.Call, func_meta) -> str:
args_count = len(node.args) - func_meta.passthrough_suffix_args_count
node_args, passthrough_suffix_args = node.args[:args_count], node.args[args_count:]
@@ -683,6 +729,52 @@ class ClickHousePrinter(BasePrinter):
finally:
self._json_function_argument_depth = depth
+ def visit_property_access(self, node: ast.PropertyAccess) -> str:
+ first_key = node.keys[0] if node.keys else None
+ with self._without_json_drop_keys(self._unmasked_key_read_type(node.expr, first_key)):
+ return super().visit_property_access(node)
+
+ def _unmasked_key_read_type(self, document: ast.Expr, first_key: object) -> ast.FieldType | None:
+ """The blob that a read of `first_key` may print without JSONDropKeysPool, or None to keep the mask.
+
+ A key that no restricted path overlaps reads the same value from the stored blob as from the masked one, so
+ the per-row UDF call only costs time. An integer first key reads by position, which dropping keys shifts.
+ Native events documents are rebuilt by the printer and keep their own masking. A user-written alias keeps the
+ mask, because ClickHouse lets the rest of the query read the aliased blob.
+ """
+ if not self.context.restricted_properties or not isinstance(first_key, str):
+ return None
+ while isinstance(document, ast.Alias) and document.hidden:
+ document = document.expr
+ if not isinstance(document, ast.Field) or not isinstance(document.type, ast.FieldType):
+ return None
+ field_type = document.type
+ resolved_field = field_type.resolve_database_field(self.context)
+ if not isinstance(resolved_field, StringJSONDatabaseField):
+ return None
+ if resolved_field.name not in RESTRICTABLE_JSON_BLOB_COLUMNS:
+ return None
+ if self._events_json_document_field(field_type) is not None:
+ return None
+ keys_to_drop = restricted_property_keys_for_table_type(
+ field_type.table_type, self.context, group_type_index=_group_type_index_of_blob(resolved_field.name)
+ )
+ if key_read_overlaps_restriction(first_key, keys_to_drop):
+ return None
+ return field_type
+
+ @contextmanager
+ def _without_json_drop_keys(self, field_type: ast.FieldType | None) -> Iterator[None]:
+ if field_type is None:
+ yield
+ return
+ previous = self._unmasked_blob_type
+ self._unmasked_blob_type = field_type
+ try:
+ yield
+ finally:
+ self._unmasked_blob_type = previous
+
def _maybe_apply_json_drop_keys(self, type: ast.FieldType, field_sql: str, mask_feature_flags: bool = False) -> str:
"""
Wraps a StringJSONDatabaseField in JSONDropKeysPool() to strip restricted property keys
@@ -693,7 +785,7 @@ class ClickHousePrinter(BasePrinter):
`$feature_flags` map, where JSONDropKeysPool cannot reach a restricted `$feature/<key>`, so the map is dropped
and merged back without the restricted flags.
"""
- if not self.context.restricted_properties:
+ if not self.context.restricted_properties or type is self._unmasked_blob_type:
return field_sql
resolved_field = type.resolve_database_field(self.context)
@@ -709,13 +801,8 @@ class ClickHousePrinter(BasePrinter):
if resolved_field.name not in RESTRICTABLE_JSON_BLOB_COLUMNS:
return field_sql
- group_type_index = None
- group_column_match = re.fullmatch(r"group(\d+)_properties", resolved_field.name)
- if group_column_match:
- group_type_index = int(group_column_match.group(1))
-
keys_to_drop = restricted_property_keys_for_table_type(
- type.table_type, self.context, group_type_index=group_type_index
+ type.table_type, self.context, group_type_index=_group_type_index_of_blob(resolved_field.name)
)
if not keys_to_drop:
return field_sql
diff --git a/posthog/hogql/printer/test/test_property_access_control.py b/posthog/hogql/printer/test/test_property_access_control.py
index bee685a1525d..935800d0f1bc 100644
--- a/posthog/hogql/printer/test/test_property_access_control.py
+++ b/posthog/hogql/printer/test/test_property_access_control.py
@@ -2,8 +2,6 @@ import uuid
from posthog.test.base import BaseTest, ClickhouseTestMixin, cleanup_materialized_columns, materialized
-from django.conf import settings
-
from parameterized import parameterized
from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
@@ -149,13 +147,56 @@ class TestRestrictPropertiesInHogQL(BaseTest):
assert "NULL AS secret_field" in sql
assert "public_field" in sql
assert "'secret_field'" not in sql
- if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA:
- assert "JSONDropKeys" not in sql
- assert not any("secret_field" in str(v) for v in values.values())
- else:
- assert "JSONDropKeys" in sql
- self._assert_value_present(values, "secret_field")
- self._assert_value_present(values, "public_field")
+ assert "JSONDropKeys" not in sql
+ assert not any("secret_field" in str(v) for v in values.values())
+
+ @parameterized.expand(
+ [
+ ("unrestricted_key", "properties.public_field", "secret_field", False),
+ ("unrestricted_nested_key", "properties.public_field.inner", "secret_field", False),
+ (
+ "json_function_with_unrestricted_key",
+ "JSONExtractString(properties, 'public_field')",
+ "secret_field",
+ False,
+ ),
+ ("json_has_unrestricted_key", "JSONHas(properties, 'public_field')", "secret_field", False),
+ ("json_function_with_user_alias", "JSONHas(properties AS p, 'public_field') AS h, p", "secret_field", True),
+ ("sibling_of_dotted_restriction", "properties.`customer.name`", "customer.ssn", False),
+ ("parent_of_dotted_restriction", "properties.customer", "customer.ssn", True),
+ ("flat_key_under_restriction", "properties.`secret_field.inner`", "secret_field", True),
+ ("json_function_reads_parent", "JSONExtractRaw(properties, 'customer')", "customer.ssn", True),
+ ("json_function_reads_by_position", "JSONExtractRaw(properties, 1)", "secret_field", True),
+ ("json_function_whole_document", "JSONExtractKeys(properties)", "secret_field", True),
+ (
+ "json_extract_with_type_only",
+ "JSONExtract(properties, 'Tuple(secret_field String)')",
+ "secret_field",
+ True,
+ ),
+ ]
+ )
+ def test_key_read_keeps_json_drop_keys_only_when_it_overlaps_a_restriction(
+ self, _name: str, expression: str, restricted_key: str, expect_mask: bool
+ ) -> None:
+ self.event_prop.name = restricted_key
+ self.event_prop.save()
+ PropertyAccessControl.objects.create(
+ team=self.team,
+ property_definition=self.event_prop,
+ access_level=PropertyAccessLevel.NONE.value,
+ )
+ context = HogQLContext(
+ team_id=self.team.pk,
+ team=self.team,
+ user=self.user,
+ enable_select_queries=True,
+ modifiers=HogQLQueryModifiers(useNewEventsSchema=False),
+ )
+ sql, _ = prepare_and_print_ast(
+ parse_select(f"SELECT {expression} AS x FROM events"), context=context, dialect="clickhouse"
+ )
+ assert ("JSONDropKeys" in sql) is expect_mask, sql
def test_user_override_allows_access(self):
# default: none
diff --git a/posthog/hogql/restricted_properties.py b/posthog/hogql/restricted_properties.py
index a87214c6701b..825282bbff33 100644
--- a/posthog/hogql/restricted_properties.py
+++ b/posthog/hogql/restricted_properties.py
@@ -149,10 +149,23 @@ def restricted_property_keys_for_table_type(
return restricted_keys
+def key_read_overlaps_restriction(first_key: str, restricted_keys: Iterable[str]) -> bool:
+ """Whether JSONDropKeys can change what a read of the top-level key `first_key` returns.
+
+ JSONDropKeys matches object keys one dot-separated segment at a time. A restricted `a` also drops the flat key
+ `a.b`, and a restricted `a.b` also filters inside the value of `a`. Any other key reads the same value from the
+ stored blob as from the masked one.
+ """
+ segments = first_key.split(".")
+ for restricted_key in restricted_keys:
+ restricted_segments = restricted_key.split(".")
+ shared = min(len(segments), len(restricted_segments))
+ if segments[:shared] == restricted_segments[:shared]:
+ return True
+ return False
+
+
def native_property_path_overlaps_restriction(property_name: str, table_type: ast.Type, context: HogQLContext) -> bool:
if not context.restricted_properties or not context.uses_new_events_schema():
return False
- return any(
- property_name == key or property_name.startswith(key + ".") or key.startswith(property_name + ".")
- for key in restricted_property_keys_for_table_type(table_type, context)
- )
+ return key_read_overlaps_restriction(property_name, restricted_property_keys_for_table_type(table_type, context))
diff --git a/posthog/hogql/transforms/test/test_property_types.py b/posthog/hogql/transforms/test/test_property_types.py
index 2449809b1466..8d34a6108752 100644
--- a/posthog/hogql/transforms/test/test_property_types.py
+++ b/posthog/hogql/transforms/test/test_property_types.py
@@ -227,7 +227,7 @@ class TestNewEventsSchemaArraySubcolumns(_NewEventsSchemaArraySubcolumnsHelpers,
{RestrictedProperty(name="$feature/secret", property_type=PropertyDefinition.Type.EVENT)},
PropertyGroupsMode.OPTIMIZED,
(
- f"JSONExtractRaw(if(empty(events.properties), '', {JSON_DROP_KEYS_CLICKHOUSE_NAME}(",
+ "JSONExtractRaw(events.properties, ",
f"JSONHas(if(empty(events.properties), '', {JSON_DROP_KEYS_CLICKHOUSE_NAME}(",
),
("properties_group_feature_flags", "mapFilter("),