:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): mask restricted person properties inside $set and $set_once (#113997)
Alex Lider736a4e51ab94
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.mdindex eeea2e7f38d6..7c8d4f3fdd3f 100644--- a/posthog/hogql/ACCESS_CONTROL.md+++ b/posthog/hogql/ACCESS_CONTROL.md@@ -201,6 +201,8 @@ Native event JSON stores a flat dotted key such as `customer.ssn` under the pathNative reads of a parent containing restricted children use the masked JSON document instead of a raw subcolumn. This also covers multi-key `JSONHas` calls with computed keys. Unrestricted siblings remain readable.A JSON function whose first key is computed per row reads the masked `properties` document, and the masked `temporary_properties` document on rows where the key names a moved property, so a restricted property stays unreadable through a computed key.+Person property restrictions also apply inside event `$set` and `$set_once` containers, including native `%2E` key spellings. Restricted values read as `NULL` and are removed from blob reads; unrestricted siblings stay readable.+The restriction set is loaded once per query in `prepare_ast_for_printing()` and cached per `(team_id, user_id)` for the request lifetime.### Coverage is per table, not per column namediff --git a/posthog/hogql/restricted_properties.py b/posthog/hogql/restricted_properties.pyindex 825282bbff33..f3e6a38d4531 100644--- a/posthog/hogql/restricted_properties.py+++ b/posthog/hogql/restricted_properties.py@@ -17,6 +17,9 @@ from posthog.constants import GROUP_TYPES_LIMITlogger = structlog.get_logger(__name__)+# Event property keys under which identify events carry person properties.+PERSON_PROPERTY_CONTAINERS: tuple[str, ...] = ("$set", "$set_once")+# JSON blob columns that hold a restrictable property class, so the printer knows which blob reads to wrap in# JSONDropKeys. Everything here must be covered by a branch in `restricted_property_keys_for_table_type`, and vice# versa — a blob whose table type maps to a property class but whose column is missing here is read unscrubbed.@@ -140,7 +143,17 @@ def restricted_property_keys_for_table_type(or restriction.group_type_index == group_type_index)}- if restricted_keys and context.uses_new_events_schema() and isinstance(table, (EventsTable, EventsPersonSubTable)):+ native_event_json = context.uses_new_events_schema() and isinstance(table, (EventsTable, EventsPersonSubTable))+ if prop_def_type == PropertyDefinition.Type.EVENT:+ # Identify events carry person properties inside `$set` / `$set_once`.+ for restriction in context.restricted_properties:+ if restriction.property_type == PropertyDefinition.Type.PERSON:+ for container in PERSON_PROPERTY_CONTAINERS:+ restricted_keys.add(f"{container}.{restriction.name}")+ if native_event_json and "." in restriction.name:+ # Native JSON escapes dots in the property name, but not in the container.+ restricted_keys.add(f"{container}.{restriction.name.replace('.', '%2E')}")+ if restricted_keys and native_event_json:# The native JSON columns store the flat key `a.b` under the path `a%2Eb`, and the printer reads a# requested `a%2Eb` from that same path. Restrict both spellings, or the encoded one reads the value.restricted_keys |= {key.replace(".", "%2E") for key in restricted_keys if "." in key}@@ -165,7 +178,22 @@ def key_read_overlaps_restriction(first_key: str, restricted_keys: Iterable[str]return False-def native_property_path_overlaps_restriction(property_name: str, table_type: ast.Type, context: HogQLContext) -> bool:- if not context.restricted_properties or not context.uses_new_events_schema():+def property_path_overlaps_restriction(property_name: str, table_type: ast.Type, context: HogQLContext) -> bool:+ """Whether `property_name` is a restricted key, sits under one, or holds one as a child.++ A parent of a restricted child must not resolve to a physical column on either schema: a materialized column for+ `$set` holds the restricted `email` inside it, and only the blob read is scrubbed.+ """+ if not context.restricted_properties:return Falsereturn key_read_overlaps_restriction(property_name, restricted_property_keys_for_table_type(table_type, context))+++def restricted_key_path(keys: Iterable[str | int], restricted_keys: set[str]) -> str | None:+ """The first dotted prefix of `keys` that names a restricted key, or None when the path is readable."""+ path = ""+ for key in keys:+ path = f"{path}.{key}" if path else str(key)+ if path in restricted_keys:+ return path+ return Nonediff --git a/posthog/hogql/test/test_restricted_properties.py b/posthog/hogql/test/test_restricted_properties.pyindex cb735f2580f0..9cd5051f973b 100644--- a/posthog/hogql/test/test_restricted_properties.py+++ b/posthog/hogql/test/test_restricted_properties.py@@ -8,7 +8,11 @@ from posthog.hogql.context import HogQLContextfrom posthog.hogql.database.database import build_database_root_nodefrom posthog.hogql.database.models import StringJSONDatabaseField, Table, TableNodefrom posthog.hogql.property_access_types import RestrictedProperty-from posthog.hogql.restricted_properties import RESTRICTABLE_JSON_BLOB_COLUMNS, restricted_property_keys_for_table_type+from posthog.hogql.restricted_properties import (+ PERSON_PROPERTY_CONTAINERS,+ RESTRICTABLE_JSON_BLOB_COLUMNS,+ restricted_property_keys_for_table_type,+)from posthog.constants import GROUP_TYPES_LIMIT@@ -25,15 +29,18 @@ _GROUP_KEYS = tuple(f"restricted_group_{index}_property" for index in range(GROU# Every catalog blob the printer masks, and the keys it drops from each. Written out rather than derived# from RESTRICTABLE_JSON_BLOB_COLUMNS, so that dropping a column name from that set fails here instead of# quietly shrinking the walk below.+# An event blob also hides the restricted person key where identify events carry it.+_EVENT_BLOB_KEYS = frozenset({_EVENT_KEY, *(f"{container}.{_PERSON_KEY}" for container in PERSON_PROPERTY_CONTAINERS)})+_MASKED_BLOBS: dict[str, frozenset[str]] = {- "events.properties (EventsTable)": frozenset({_EVENT_KEY}),- "ai_events.properties (AiEventsTable)": frozenset({_EVENT_KEY}),+ "events.properties (EventsTable)": _EVENT_BLOB_KEYS,+ "ai_events.properties (AiEventsTable)": _EVENT_BLOB_KEYS,"events.person_properties (EventsPersonSubTable)": frozenset({_PERSON_KEY}),**{f"events.group{index}_properties (EventsGroupSubTable)": frozenset({_GROUP_KEYS[index]})for index in range(GROUP_TYPES_LIMIT)},- "flag_evaluations.properties (FlagEvaluationsTable)": frozenset({_EVENT_KEY}),+ "flag_evaluations.properties (FlagEvaluationsTable)": _EVENT_BLOB_KEYS,"flag_evaluations.person_properties (FlagEvaluationsPersonSubTable)": frozenset({_PERSON_KEY}),"persons.properties (PersonsTable)": frozenset({_PERSON_KEY}),"raw_persons.properties (RawPersonsTable)": frozenset({_PERSON_KEY}),@@ -135,4 +142,8 @@ def test_every_restrictable_blob_column_is_masked_with_the_keys_of_its_property_if use_new_events_schema:for label in ("events.properties (EventsTable)", "events.person_properties (EventsPersonSubTable)"):expected[label] |= {"$unparseable_properties"}+ # The native schema also restricts the `%2E` spelling of every dotted key.+ expected["events.properties (EventsTable)"] |= {+ f"{container}%2E{_PERSON_KEY}" for container in PERSON_PROPERTY_CONTAINERS+ }assert masked == expecteddiff --git a/posthog/hogql/transforms/clickhouse_property_resolution.py b/posthog/hogql/transforms/clickhouse_property_resolution.pyindex b60be81dbb09..75ba5a849855 100644--- a/posthog/hogql/transforms/clickhouse_property_resolution.py+++ b/posthog/hogql/transforms/clickhouse_property_resolution.py@@ -44,8 +44,9 @@ from posthog.hogql.printer.base import resolve_field_typefrom posthog.hogql.printer.clickhouse import AI_BLOOM_FILTER_PROPERTIES, COLUMNS_WITH_HACKY_OPTIMIZED_NULL_HANDLINGfrom posthog.hogql.restricted_properties import (mirrored_property_for_column,- native_property_path_overlaps_restriction,+ property_path_overlaps_restriction,restricted_feature_flag_names,+ restricted_key_path,restricted_property_keys_for_table_type,)from posthog.hogql.type_system import (@@ -131,9 +132,7 @@ def resolve_materialized_property_source(# read, comparison, key-existence). The column holds the raw value, so a comparison like `WHERE properties.x = 'y'`# could otherwise read it and probe the value. Declining here makes the comparison optimizers fall back; the read# itself becomes a constant NULL in `_substitute_value_read`.- if property_name in restricted_property_keys_for_table_type(- field_type.table_type, context- ) or native_property_path_overlaps_restriction(property_name, field_type.table_type, context):+ if property_path_overlaps_restriction(property_name, field_type.table_type, context):return Nonetable_type = _unwrap_to_table_type(field_type)@@ -195,7 +194,7 @@ def resolve_materialized_property_source(def resolve_json_subcolumn_source(field_type: ast.FieldType, table_name: str, field_name: str, property_name: str, context: HogQLContext) -> MaterializedPropertySource | None:- if native_property_path_overlaps_restriction(property_name, field_type.table_type, context):+ if property_path_overlaps_restriction(property_name, field_type.table_type, context):return Noneif not context.uses_new_events_schema():return None@@ -934,7 +933,12 @@ def _substitute_value_read(node: ast.PropertyAccess, context: HogQLContext) -> a# directly and skip the wasted drop-then-extract. (The column resolvers also decline, so comparisons over a# restricted property never read the backing column either; their operand falls through to this same NULL.)source_property = _mirrored_source_property(field_type, context)- if (source_property or first_key) in restricted_property_keys_for_table_type(field_type.table_type, context):+ restricted_keys = restricted_property_keys_for_table_type(field_type.table_type, context)+ if (+ source_property in restricted_keys+ if source_property is not None+ else restricted_key_path(node.keys, restricted_keys) is not None+ ):_record_property_usage(context, None)return ast.Constant(value=None, type=ast.StringType(nullable=True))@@ -1841,7 +1845,7 @@ class ClickHousePropertyResolver(CloningVisitor):first_key = first_key_arg.valueif first_key in restricted_property_keys_for_table_type(field_type.table_type, self.context):return ast.Constant(value=False, type=ast.BooleanType(nullable=False))- if native_property_path_overlaps_restriction(first_key, field_type.table_type, self.context):+ if property_path_overlaps_restriction(first_key, field_type.table_type, self.context):# Read the masked document so nested and computed keys cannot inspect a restricted child.if not _is_moved_events_property(field_type, first_key, self.context):return Nonediff --git a/posthog/hogql/transforms/test/test_property_types.py b/posthog/hogql/transforms/test/test_property_types.pyindex 1a3a72976ab6..52af7059d9d7 100644--- a/posthog/hogql/transforms/test/test_property_types.py+++ b/posthog/hogql/transforms/test/test_property_types.py@@ -682,6 +682,65 @@ class TestNewEventsSchemaArraySubcolumnsClickhouse(_NewEventsSchemaArraySubcolum)assert row == ('{"project":"visible"}', None, "visible", 0, 0, 1)+ def test_person_restriction_masks_set_containers_on_events(self) -> None:+ context = self._context()+ context.restricted_properties = {+ RestrictedProperty(name=name, property_type=PropertyDefinition.Type.PERSON)+ for name in ("email", "user.email")+ }+ query = parse_select(+ """SELECT properties.$set,+ properties.$set.email,+ properties.$set."user%2Eemail" AS dotted_set,+ properties.$set.name,+ properties.$set_once.email,+ properties.$set_once."user%2Eemail" AS dotted_set_once,+ properties.email,+ JSONHas(properties, '$set', 'email'),+ JSONHas(properties, '$set', 'user%2Eemail'),+ JSONExtractString(properties, '$set_once', 'email')+ FROM events"""+ )+ with patch("posthog.hogql.printer.utils.build_property_swapper"):+ printed, _ = prepare_and_print_ast(query, context, "clickhouse")+ [row] = sync_execute(+ """WITH events_json AS (+ SELECT 1 AS team_id, '$identify' AS event,+ CAST(%(document)s, %(event_type)s) AS properties,+ CAST('{}', %(person_type)s) AS person_properties,+ CAST(%(temporary_document)s, %(temporary_type)s) AS temporary_properties+ ) """+ + printed,+ {+ **context.values,+ "document": '{"email":"event-level"}',+ # The native cleaner stores $set and $set_once in temporary_properties, so a read of either+ # container comes from that document.+ "temporary_document": '{"$set":{"email":"hidden","user%2Eemail":"hidden","name":"visible"},"$set_once":{"email":"hidden","user%2Eemail":"hidden"}}',+ "event_type": EVENTS_PROPERTIES_JSON_TYPE(),+ "person_type": PERSON_PROPERTIES_JSON_TYPE(),+ "temporary_type": TEMPORARY_PROPERTIES_JSON_TYPE,+ },+ )+ # `email` as an event property is a different property and stays readable.+ assert row == ('{"name":"visible"}', None, None, "visible", None, None, "event-level", 0, 0, "")++ def test_person_restriction_masks_set_containers_on_legacy_events(self) -> None:+ context = self._context(use_new_events_schema=False)+ context.restricted_properties = {RestrictedProperty(name="email", property_type=PropertyDefinition.Type.PERSON)}+ query = parse_select("SELECT properties.$set.email, properties.$set, properties.email FROM events")+ with patch("posthog.hogql.printer.utils.build_property_swapper"):+ printed, _ = prepare_and_print_ast(query, context, "clickhouse")+ select_list = printed.split("FROM")[0]+ # The nested read is a constant, never an extract that could reach the raw blob.+ assert select_list.startswith("SELECT NULL AS `$set__email`,"), printed+ # The container read comes from the scrubbed blob, which drops the nested path.+ assert JSON_DROP_KEYS_CLICKHOUSE_NAME in select_list, printed+ dropped = [value for value in context.values.values() if isinstance(value, list)]+ assert any("$set.email" in value and "$set_once.email" in value for value in dropped), context.values+ # The event-level `email` is unrestricted and still read from the blob.+ assert "email" not in {key for value in dropped for key in value}, context.values+class TestPropertyTypes(BaseTest):snapshot: Anydiff --git a/posthog/hogql_queries/events_query_runner.py b/posthog/hogql_queries/events_query_runner.pyindex 7c9f684e7908..ebbcc07348dd 100644--- a/posthog/hogql_queries/events_query_runner.py+++ b/posthog/hogql_queries/events_query_runner.py@@ -42,6 +42,7 @@ from posthog.hogql.property import (steps_to_expr,)from posthog.hogql.query import execute_hogql_query+from posthog.hogql.restricted_properties import PERSON_PROPERTY_CONTAINERSfrom posthog.hogql.visitor import TraversingVisitorfrom posthog.api.element import ElementSerializer@@ -385,6 +386,14 @@ class EventsQueryRunner(AnalyticsQueryRunner[EventsQueryResponse]):# ``properties.<name>`` on the events table.if len(chain) >= 2 and chain[0] == "properties" and chain[1] in restricted_event_props:raise ResolutionError(f"Access to property '{chain[1]}' is restricted")+ # ``properties.$set.<name>`` carries the person property on an identify event.+ if (+ len(chain) >= 3+ and chain[0] == "properties"+ and chain[1] in PERSON_PROPERTY_CONTAINERS+ and chain[2] in restricted_person_props+ ):+ raise ResolutionError(f"Access to property '{chain[2]}' is restricted")# ``person.properties.<name>`` (or ``poe.properties.<name>``) on the joined person.if (len(chain) >= 3diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.pyindex e4a7a2cc11fe..f1a20468a6e7 100644--- a/posthog/hogql_queries/query_runner.py+++ b/posthog/hogql_queries/query_runner.py@@ -3226,7 +3226,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):payload["restricted_properties"] = restricted# A cache hit skips masking. Bump this when masking expands to new tables or mirror columns# so restricted results cached before that change are invalidated.- payload["property_access_control_version"] = 1+ payload["property_access_control_version"] = 2# Vary the cache key by the events-retention floor: a cache hit returns before the printer applies the floor,# so without this a result cached pre-enforcement (or at a longer period) would keep surfacing events pastdiff --git a/posthog/hogql_queries/test/test_events_query_runner.py b/posthog/hogql_queries/test/test_events_query_runner.pyindex 9736d01c6cc3..52d0781a404a 100644--- a/posthog/hogql_queries/test/test_events_query_runner.py+++ b/posthog/hogql_queries/test/test_events_query_runner.py@@ -1779,12 +1779,18 @@ class TestEventsQueryRunner(ClickhouseTestMixin, APIBaseTest):assert "email" not in person_data["properties"]assert "name" in person_data["properties"]+ @parameterized.expand(+ [+ ("event_property", PropertyDefinition.Type.EVENT, "secret_field", "properties.secret_field"),+ ("person_property_in_set", PropertyDefinition.Type.PERSON, "secret_field", "properties.$set.secret_field"),+ ]+ )@time_machine.travel("2020-01-11T12:00:05Z", tick=False)- def test_restricted_event_property_in_select_raises_error(self):+ def test_restricted_property_in_select_raises_error(+ self, _name: str, property_type: int, property_name: str, select: str+ ) -> None:from posthog.hogql.errors import ResolutionError- from posthog.models import PropertyDefinition-from products.access_control.backend.models.property_access_control import PropertyAccessControlfrom products.access_control.backend.property_access_control import PropertyAccessLevel@@ -1795,14 +1801,14 @@ class TestEventsQueryRunner(ClickhouseTestMixin, APIBaseTest):event="$pageview",distinct_id="p1",timestamp="2020-01-11T12:00:01Z",- properties={"secret_field": "hidden"},+ properties={"secret_field": "hidden", "$set": {"secret_field": "hidden"}},)flush_persons_and_events()prop_def = PropertyDefinition.objects.create(team=self.team,- name="secret_field",- type=PropertyDefinition.Type.EVENT,+ name=property_name,+ type=property_type,)PropertyAccessControl.objects.create(team=self.team,@@ -1810,7 +1816,7 @@ class TestEventsQueryRunner(ClickhouseTestMixin, APIBaseTest):access_level=PropertyAccessLevel.NONE.value,)- query = EventsQuery(select=["properties.secret_field"], after="2020-01-10")+ query = EventsQuery(select=[select], after="2020-01-10")runner = EventsQueryRunner(query=query, team=self.team, user=self.user)with self.assertRaises(ResolutionError):runner.run()