Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): mask restricted person properties inside $set and $set_once (#113997)

Alex Lider736a4e51ab94
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.md
index eeea2e7f38d6..7c8d4f3fdd3f 100644
--- a/posthog/hogql/ACCESS_CONTROL.md
+++ b/posthog/hogql/ACCESS_CONTROL.md
@@ -201,6 +201,8 @@ Native event JSON stores a flat dotted key such as `customer.ssn` under the path
Native reads of a parent containing restricted children use the masked JSON document instead of a raw subcolumn. This also covers multi-key `JSONHas` calls with computed keys. Unrestricted siblings remain readable.
A JSON function whose first key is computed per row reads the masked `properties` document, and the masked `temporary_properties` document on rows where the key names a moved property, so a restricted property stays unreadable through a computed key.
+Person property restrictions also apply inside event `$set` and `$set_once` containers, including native `%2E` key spellings. Restricted values read as `NULL` and are removed from blob reads; unrestricted siblings stay readable.
+
The restriction set is loaded once per query in `prepare_ast_for_printing()` and cached per `(team_id, user_id)` for the request lifetime.
### Coverage is per table, not per column name
diff --git a/posthog/hogql/restricted_properties.py b/posthog/hogql/restricted_properties.py
index 825282bbff33..f3e6a38d4531 100644
--- a/posthog/hogql/restricted_properties.py
+++ b/posthog/hogql/restricted_properties.py
@@ -17,6 +17,9 @@ from posthog.constants import GROUP_TYPES_LIMIT
logger = structlog.get_logger(__name__)
+# Event property keys under which identify events carry person properties.
+PERSON_PROPERTY_CONTAINERS: tuple[str, ...] = ("$set", "$set_once")
+
# JSON blob columns that hold a restrictable property class, so the printer knows which blob reads to wrap in
# JSONDropKeys. Everything here must be covered by a branch in `restricted_property_keys_for_table_type`, and vice
# versa — a blob whose table type maps to a property class but whose column is missing here is read unscrubbed.
@@ -140,7 +143,17 @@ def restricted_property_keys_for_table_type(
or restriction.group_type_index == group_type_index
)
}
- if restricted_keys and context.uses_new_events_schema() and isinstance(table, (EventsTable, EventsPersonSubTable)):
+ native_event_json = context.uses_new_events_schema() and isinstance(table, (EventsTable, EventsPersonSubTable))
+ if prop_def_type == PropertyDefinition.Type.EVENT:
+ # Identify events carry person properties inside `$set` / `$set_once`.
+ for restriction in context.restricted_properties:
+ if restriction.property_type == PropertyDefinition.Type.PERSON:
+ for container in PERSON_PROPERTY_CONTAINERS:
+ restricted_keys.add(f"{container}.{restriction.name}")
+ if native_event_json and "." in restriction.name:
+ # Native JSON escapes dots in the property name, but not in the container.
+ restricted_keys.add(f"{container}.{restriction.name.replace('.', '%2E')}")
+ if restricted_keys and native_event_json:
# The native JSON columns store the flat key `a.b` under the path `a%2Eb`, and the printer reads a
# requested `a%2Eb` from that same path. Restrict both spellings, or the encoded one reads the value.
restricted_keys |= {key.replace(".", "%2E") for key in restricted_keys if "." in key}
@@ -165,7 +178,22 @@ def key_read_overlaps_restriction(first_key: str, restricted_keys: Iterable[str]
return False
-def native_property_path_overlaps_restriction(property_name: str, table_type: ast.Type, context: HogQLContext) -> bool:
- if not context.restricted_properties or not context.uses_new_events_schema():
+def property_path_overlaps_restriction(property_name: str, table_type: ast.Type, context: HogQLContext) -> bool:
+ """Whether `property_name` is a restricted key, sits under one, or holds one as a child.
+
+ A parent of a restricted child must not resolve to a physical column on either schema: a materialized column for
+ `$set` holds the restricted `email` inside it, and only the blob read is scrubbed.
+ """
+ if not context.restricted_properties:
return False
return key_read_overlaps_restriction(property_name, restricted_property_keys_for_table_type(table_type, context))
+
+
+def restricted_key_path(keys: Iterable[str | int], restricted_keys: set[str]) -> str | None:
+ """The first dotted prefix of `keys` that names a restricted key, or None when the path is readable."""
+ path = ""
+ for key in keys:
+ path = f"{path}.{key}" if path else str(key)
+ if path in restricted_keys:
+ return path
+ return None
diff --git a/posthog/hogql/test/test_restricted_properties.py b/posthog/hogql/test/test_restricted_properties.py
index cb735f2580f0..9cd5051f973b 100644
--- a/posthog/hogql/test/test_restricted_properties.py
+++ b/posthog/hogql/test/test_restricted_properties.py
@@ -8,7 +8,11 @@ from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import build_database_root_node
from posthog.hogql.database.models import StringJSONDatabaseField, Table, TableNode
from posthog.hogql.property_access_types import RestrictedProperty
-from posthog.hogql.restricted_properties import RESTRICTABLE_JSON_BLOB_COLUMNS, restricted_property_keys_for_table_type
+from posthog.hogql.restricted_properties import (
+ PERSON_PROPERTY_CONTAINERS,
+ RESTRICTABLE_JSON_BLOB_COLUMNS,
+ restricted_property_keys_for_table_type,
+)
from posthog.constants import GROUP_TYPES_LIMIT
@@ -25,15 +29,18 @@ _GROUP_KEYS = tuple(f"restricted_group_{index}_property" for index in range(GROU
# Every catalog blob the printer masks, and the keys it drops from each. Written out rather than derived
# from RESTRICTABLE_JSON_BLOB_COLUMNS, so that dropping a column name from that set fails here instead of
# quietly shrinking the walk below.
+# An event blob also hides the restricted person key where identify events carry it.
+_EVENT_BLOB_KEYS = frozenset({_EVENT_KEY, *(f"{container}.{_PERSON_KEY}" for container in PERSON_PROPERTY_CONTAINERS)})
+
_MASKED_BLOBS: dict[str, frozenset[str]] = {
- "events.properties (EventsTable)": frozenset({_EVENT_KEY}),
- "ai_events.properties (AiEventsTable)": frozenset({_EVENT_KEY}),
+ "events.properties (EventsTable)": _EVENT_BLOB_KEYS,
+ "ai_events.properties (AiEventsTable)": _EVENT_BLOB_KEYS,
"events.person_properties (EventsPersonSubTable)": frozenset({_PERSON_KEY}),
**{
f"events.group{index}_properties (EventsGroupSubTable)": frozenset({_GROUP_KEYS[index]})
for index in range(GROUP_TYPES_LIMIT)
},
- "flag_evaluations.properties (FlagEvaluationsTable)": frozenset({_EVENT_KEY}),
+ "flag_evaluations.properties (FlagEvaluationsTable)": _EVENT_BLOB_KEYS,
"flag_evaluations.person_properties (FlagEvaluationsPersonSubTable)": frozenset({_PERSON_KEY}),
"persons.properties (PersonsTable)": frozenset({_PERSON_KEY}),
"raw_persons.properties (RawPersonsTable)": frozenset({_PERSON_KEY}),
@@ -135,4 +142,8 @@ def test_every_restrictable_blob_column_is_masked_with_the_keys_of_its_property_
if use_new_events_schema:
for label in ("events.properties (EventsTable)", "events.person_properties (EventsPersonSubTable)"):
expected[label] |= {"$unparseable_properties"}
+ # The native schema also restricts the `%2E` spelling of every dotted key.
+ expected["events.properties (EventsTable)"] |= {
+ f"{container}%2E{_PERSON_KEY}" for container in PERSON_PROPERTY_CONTAINERS
+ }
assert masked == expected
diff --git a/posthog/hogql/transforms/clickhouse_property_resolution.py b/posthog/hogql/transforms/clickhouse_property_resolution.py
index b60be81dbb09..75ba5a849855 100644
--- a/posthog/hogql/transforms/clickhouse_property_resolution.py
+++ b/posthog/hogql/transforms/clickhouse_property_resolution.py
@@ -44,8 +44,9 @@ from posthog.hogql.printer.base import resolve_field_type
from posthog.hogql.printer.clickhouse import AI_BLOOM_FILTER_PROPERTIES, COLUMNS_WITH_HACKY_OPTIMIZED_NULL_HANDLING
from posthog.hogql.restricted_properties import (
mirrored_property_for_column,
- native_property_path_overlaps_restriction,
+ property_path_overlaps_restriction,
restricted_feature_flag_names,
+ restricted_key_path,
restricted_property_keys_for_table_type,
)
from posthog.hogql.type_system import (
@@ -131,9 +132,7 @@ def resolve_materialized_property_source(
# read, comparison, key-existence). The column holds the raw value, so a comparison like `WHERE properties.x = 'y'`
# could otherwise read it and probe the value. Declining here makes the comparison optimizers fall back; the read
# itself becomes a constant NULL in `_substitute_value_read`.
- if property_name in restricted_property_keys_for_table_type(
- field_type.table_type, context
- ) or native_property_path_overlaps_restriction(property_name, field_type.table_type, context):
+ if property_path_overlaps_restriction(property_name, field_type.table_type, context):
return None
table_type = _unwrap_to_table_type(field_type)
@@ -195,7 +194,7 @@ def resolve_materialized_property_source(
def resolve_json_subcolumn_source(
field_type: ast.FieldType, table_name: str, field_name: str, property_name: str, context: HogQLContext
) -> MaterializedPropertySource | None:
- if native_property_path_overlaps_restriction(property_name, field_type.table_type, context):
+ if property_path_overlaps_restriction(property_name, field_type.table_type, context):
return None
if not context.uses_new_events_schema():
return None
@@ -934,7 +933,12 @@ def _substitute_value_read(node: ast.PropertyAccess, context: HogQLContext) -> a
# directly and skip the wasted drop-then-extract. (The column resolvers also decline, so comparisons over a
# restricted property never read the backing column either; their operand falls through to this same NULL.)
source_property = _mirrored_source_property(field_type, context)
- if (source_property or first_key) in restricted_property_keys_for_table_type(field_type.table_type, context):
+ restricted_keys = restricted_property_keys_for_table_type(field_type.table_type, context)
+ if (
+ source_property in restricted_keys
+ if source_property is not None
+ else restricted_key_path(node.keys, restricted_keys) is not None
+ ):
_record_property_usage(context, None)
return ast.Constant(value=None, type=ast.StringType(nullable=True))
@@ -1841,7 +1845,7 @@ class ClickHousePropertyResolver(CloningVisitor):
first_key = first_key_arg.value
if first_key in restricted_property_keys_for_table_type(field_type.table_type, self.context):
return ast.Constant(value=False, type=ast.BooleanType(nullable=False))
- if native_property_path_overlaps_restriction(first_key, field_type.table_type, self.context):
+ if property_path_overlaps_restriction(first_key, field_type.table_type, self.context):
# Read the masked document so nested and computed keys cannot inspect a restricted child.
if not _is_moved_events_property(field_type, first_key, self.context):
return None
diff --git a/posthog/hogql/transforms/test/test_property_types.py b/posthog/hogql/transforms/test/test_property_types.py
index 1a3a72976ab6..52af7059d9d7 100644
--- a/posthog/hogql/transforms/test/test_property_types.py
+++ b/posthog/hogql/transforms/test/test_property_types.py
@@ -682,6 +682,65 @@ class TestNewEventsSchemaArraySubcolumnsClickhouse(_NewEventsSchemaArraySubcolum
)
assert row == ('{"project":"visible"}', None, "visible", 0, 0, 1)
+ def test_person_restriction_masks_set_containers_on_events(self) -> None:
+ context = self._context()
+ context.restricted_properties = {
+ RestrictedProperty(name=name, property_type=PropertyDefinition.Type.PERSON)
+ for name in ("email", "user.email")
+ }
+ query = parse_select(
+ """SELECT properties.$set,
+ properties.$set.email,
+ properties.$set."user%2Eemail" AS dotted_set,
+ properties.$set.name,
+ properties.$set_once.email,
+ properties.$set_once."user%2Eemail" AS dotted_set_once,
+ properties.email,
+ JSONHas(properties, '$set', 'email'),
+ JSONHas(properties, '$set', 'user%2Eemail'),
+ JSONExtractString(properties, '$set_once', 'email')
+ FROM events"""
+ )
+ with patch("posthog.hogql.printer.utils.build_property_swapper"):
+ printed, _ = prepare_and_print_ast(query, context, "clickhouse")
+ [row] = sync_execute(
+ """WITH events_json AS (
+ SELECT 1 AS team_id, '$identify' AS event,
+ CAST(%(document)s, %(event_type)s) AS properties,
+ CAST('{}', %(person_type)s) AS person_properties,
+ CAST(%(temporary_document)s, %(temporary_type)s) AS temporary_properties
+ ) """
+ + printed,
+ {
+ **context.values,
+ "document": '{"email":"event-level"}',
+ # The native cleaner stores $set and $set_once in temporary_properties, so a read of either
+ # container comes from that document.
+ "temporary_document": '{"$set":{"email":"hidden","user%2Eemail":"hidden","name":"visible"},"$set_once":{"email":"hidden","user%2Eemail":"hidden"}}',
+ "event_type": EVENTS_PROPERTIES_JSON_TYPE(),
+ "person_type": PERSON_PROPERTIES_JSON_TYPE(),
+ "temporary_type": TEMPORARY_PROPERTIES_JSON_TYPE,
+ },
+ )
+ # `email` as an event property is a different property and stays readable.
+ assert row == ('{"name":"visible"}', None, None, "visible", None, None, "event-level", 0, 0, "")
+
+ def test_person_restriction_masks_set_containers_on_legacy_events(self) -> None:
+ context = self._context(use_new_events_schema=False)
+ context.restricted_properties = {RestrictedProperty(name="email", property_type=PropertyDefinition.Type.PERSON)}
+ query = parse_select("SELECT properties.$set.email, properties.$set, properties.email FROM events")
+ with patch("posthog.hogql.printer.utils.build_property_swapper"):
+ printed, _ = prepare_and_print_ast(query, context, "clickhouse")
+ select_list = printed.split("FROM")[0]
+ # The nested read is a constant, never an extract that could reach the raw blob.
+ assert select_list.startswith("SELECT NULL AS `$set__email`,"), printed
+ # The container read comes from the scrubbed blob, which drops the nested path.
+ assert JSON_DROP_KEYS_CLICKHOUSE_NAME in select_list, printed
+ dropped = [value for value in context.values.values() if isinstance(value, list)]
+ assert any("$set.email" in value and "$set_once.email" in value for value in dropped), context.values
+ # The event-level `email` is unrestricted and still read from the blob.
+ assert "email" not in {key for value in dropped for key in value}, context.values
+
class TestPropertyTypes(BaseTest):
snapshot: Any
diff --git a/posthog/hogql_queries/events_query_runner.py b/posthog/hogql_queries/events_query_runner.py
index 7c9f684e7908..ebbcc07348dd 100644
--- a/posthog/hogql_queries/events_query_runner.py
+++ b/posthog/hogql_queries/events_query_runner.py
@@ -42,6 +42,7 @@ from posthog.hogql.property import (
steps_to_expr,
)
from posthog.hogql.query import execute_hogql_query
+from posthog.hogql.restricted_properties import PERSON_PROPERTY_CONTAINERS
from posthog.hogql.visitor import TraversingVisitor
from posthog.api.element import ElementSerializer
@@ -385,6 +386,14 @@ class EventsQueryRunner(AnalyticsQueryRunner[EventsQueryResponse]):
# ``properties.<name>`` on the events table.
if len(chain) >= 2 and chain[0] == "properties" and chain[1] in restricted_event_props:
raise ResolutionError(f"Access to property '{chain[1]}' is restricted")
+ # ``properties.$set.<name>`` carries the person property on an identify event.
+ if (
+ len(chain) >= 3
+ and chain[0] == "properties"
+ and chain[1] in PERSON_PROPERTY_CONTAINERS
+ and chain[2] in restricted_person_props
+ ):
+ raise ResolutionError(f"Access to property '{chain[2]}' is restricted")
# ``person.properties.<name>`` (or ``poe.properties.<name>``) on the joined person.
if (
len(chain) >= 3
diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.py
index e4a7a2cc11fe..f1a20468a6e7 100644
--- a/posthog/hogql_queries/query_runner.py
+++ b/posthog/hogql_queries/query_runner.py
@@ -3226,7 +3226,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
payload["restricted_properties"] = restricted
# A cache hit skips masking. Bump this when masking expands to new tables or mirror columns
# so restricted results cached before that change are invalidated.
- payload["property_access_control_version"] = 1
+ payload["property_access_control_version"] = 2
# Vary the cache key by the events-retention floor: a cache hit returns before the printer applies the floor,
# so without this a result cached pre-enforcement (or at a longer period) would keep surfacing events past
diff --git a/posthog/hogql_queries/test/test_events_query_runner.py b/posthog/hogql_queries/test/test_events_query_runner.py
index 9736d01c6cc3..52d0781a404a 100644
--- a/posthog/hogql_queries/test/test_events_query_runner.py
+++ b/posthog/hogql_queries/test/test_events_query_runner.py
@@ -1779,12 +1779,18 @@ class TestEventsQueryRunner(ClickhouseTestMixin, APIBaseTest):
assert "email" not in person_data["properties"]
assert "name" in person_data["properties"]
+ @parameterized.expand(
+ [
+ ("event_property", PropertyDefinition.Type.EVENT, "secret_field", "properties.secret_field"),
+ ("person_property_in_set", PropertyDefinition.Type.PERSON, "secret_field", "properties.$set.secret_field"),
+ ]
+ )
@time_machine.travel("2020-01-11T12:00:05Z", tick=False)
- def test_restricted_event_property_in_select_raises_error(self):
+ def test_restricted_property_in_select_raises_error(
+ self, _name: str, property_type: int, property_name: str, select: str
+ ) -> None:
from posthog.hogql.errors import ResolutionError
- from posthog.models import PropertyDefinition
-
from products.access_control.backend.models.property_access_control import PropertyAccessControl
from products.access_control.backend.property_access_control import PropertyAccessLevel
@@ -1795,14 +1801,14 @@ class TestEventsQueryRunner(ClickhouseTestMixin, APIBaseTest):
event="$pageview",
distinct_id="p1",
timestamp="2020-01-11T12:00:01Z",
- properties={"secret_field": "hidden"},
+ properties={"secret_field": "hidden", "$set": {"secret_field": "hidden"}},
)
flush_persons_and_events()
prop_def = PropertyDefinition.objects.create(
team=self.team,
- name="secret_field",
- type=PropertyDefinition.Type.EVENT,
+ name=property_name,
+ type=property_type,
)
PropertyAccessControl.objects.create(
team=self.team,
@@ -1810,7 +1816,7 @@ class TestEventsQueryRunner(ClickhouseTestMixin, APIBaseTest):
access_level=PropertyAccessLevel.NONE.value,
)
- query = EventsQuery(select=["properties.secret_field"], after="2020-01-10")
+ query = EventsQuery(select=[select], after="2020-01-10")
runner = EventsQueryRunner(query=query, team=self.team, user=self.user)
with self.assertRaises(ResolutionError):
runner.run()