:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): strip restricted property keys with the pooled udf (#120479)
Rory Shanksf03f94bf040c
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.mdindex 64777bc540b0..90f981b8119c 100644--- a/posthog/hogql/ACCESS_CONTROL.md+++ b/posthog/hogql/ACCESS_CONTROL.md@@ -185,7 +185,7 @@ Unlike levels 1 and 2, restricted properties do not error.They're masked when the query is printed to ClickHouse SQL, so a restricted read is indistinguishable from a property that was never set (anti-enumeration):- **Explicit reads** (`properties.email`) are replaced with `NULL`, and the resolver refuses to back them with a materialized column — `ClickHousePropertyResolver` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.-- **Whole-blob reads** (`SELECT properties` or `SELECT *`) have the restricted keys stripped from the returned JSON via `JSONDropKeys(...)` — `ClickHousePrinter._maybe_apply_json_drop_keys()` in `posthog/hogql/printer/clickhouse.py`.+- **Whole-blob reads** (`SELECT properties` or `SELECT *`) have the restricted keys stripped from the returned JSON via `JSONDropKeysPool(...)` — `ClickHousePrinter._maybe_apply_json_drop_keys()` in `posthog/hogql/printer/clickhouse.py`.Group restrictions retain their group type index, so a same-named property on another group type stays readable. The masking also applies to the Postgres-backed `system.groups.group_properties` field.diff --git a/posthog/hogql/functions/udfs.py b/posthog/hogql/functions/udfs.pyindex 368509dea093..3e80f42ca13c 100644--- a/posthog/hogql/functions/udfs.py+++ b/posthog/hogql/functions/udfs.py@@ -28,9 +28,11 @@ UDFS: dict[str, HogQLFunctionMeta] = {),}-# JSONDropKeys is an executable UDF like the funnel UDFs, but it is printer-internal (restricted-property blob-# stripping), not HogQL-exposed, so it lives outside UDFS. Its deployed name is versioned the same way.-JSON_DROP_KEYS_CLICKHOUSE_NAME = "JSONDropKeys"+# JSONDropKeysPool strips restricted keys from a JSON blob. It is printer-internal, not HogQL-exposed, so it lives+# outside UDFS. It is an executable_pool UDF in UNVERSIONED_FUNCTIONS, so its deployed name has no version suffix.+# Do not use JSONDropKeys here: it takes the keys as a command-line parameter, which forces the `executable` type,+# and ClickHouse then starts a new process for every block.+JSON_DROP_KEYS_CLICKHOUSE_NAME = "JSONDropKeysPool"# We want CI to fail if there is a breaking change and the version hasn't been incrementedif is_cloud() or is_ci():@@ -38,4 +40,3 @@ if is_cloud() or is_ci():for v in UDFS.values():v.clickhouse_name = augment_function_name(v.clickhouse_name)- JSON_DROP_KEYS_CLICKHOUSE_NAME = augment_function_name(JSON_DROP_KEYS_CLICKHOUSE_NAME)diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.pyindex 5acd6bd92990..92bb29e3e75d 100644--- a/posthog/hogql/printer/clickhouse.py+++ b/posthog/hogql/printer/clickhouse.py@@ -683,13 +683,13 @@ class ClickHousePrinter(BasePrinter):def _maybe_apply_json_drop_keys(self, type: ast.FieldType, field_sql: str, mask_feature_flags: bool = False) -> str:"""- Wraps a StringJSONDatabaseField in JSONDropKeys() to strip restricted property keys+ Wraps a StringJSONDatabaseField in JSONDropKeysPool() to strip restricted property keyswhen the raw JSON blob is selected directly (e.g., `SELECT properties FROM events`).`mask_feature_flags` is for an events_json `properties` column printed as stored rather than as the rebuiltdocument, which happens under `toJSONString(if(..., properties, ...))`. That document holds the flags in the- `$feature_flags` map, where JSONDropKeys cannot reach a restricted `$feature/<key>`, so the map is dropped and- merged back without the restricted flags.+ `$feature_flags` map, where JSONDropKeysPool cannot reach a restricted `$feature/<key>`, so the map is dropped+ and merged back without the restricted flags."""if not self.context.restricted_properties:return field_sql@@ -701,7 +701,7 @@ class ClickHousePrinter(BasePrinter):# Use the resolved DB column name, not ``type.name``. With column-alias table syntax# (``FROM events AS e(uuid, event, ..., p)``) the AST field name is the alias (``p``),# but ClickHouse resolves it back to the original column. Comparing ``type.name`` here- # would incorrectly skip JSONDropKeys wrapping for the aliased ``properties`` column.+ # would incorrectly skip JSONDropKeysPool wrapping for the aliased ``properties`` column.# ``person_properties`` is the underlying DB column for ``EventsPersonSubTable.properties``# (PoE mode); it is also a JSON blob that must be stripped of restricted person-property keys.if resolved_field.name not in RESTRICTABLE_JSON_BLOB_COLUMNS:@@ -733,13 +733,19 @@ class ClickHousePrinter(BasePrinter):}keys_placeholder = self.context.add_sensitive_value(sorted(keys_to_drop))- stripped = f"{JSON_DROP_KEYS_CLICKHOUSE_NAME}({keys_placeholder})({field_sql})"- if not is_events_json_document:- # A String JSON column holds '' when its producer omits the value.- # The v12 json_drop_keys_udf binary fails to parse ''. Cloud and CI call it as JSONDropKeys_v12.- # ClickHouse short-circuits `if`. The UDF therefore receives only the rows that are not empty.+ if is_events_json_document:# A native events document is a JSON column or a document that concat rebuilds, so it is never ''.- stripped = f"if(empty({field_sql}), '', {stripped})"+ # The UDF reads RowBinary, which encodes a JSON column differently from a String. Without toString the+ # worker misreads the row and waits for more input until ClickHouse times out the pipe.+ stripped = f"{JSON_DROP_KEYS_CLICKHOUSE_NAME}(toString({field_sql}), {keys_placeholder})"+ else:+ # A String JSON column holds '' when its producer omits the value, and the UDF fails to parse ''.+ # ClickHouse can evaluate both branches of `if`, so the UDF gets '{}' in place of '' and the outer+ # `if` discards that result.+ guarded_sql = f"if(empty({field_sql}), '{{}}', {field_sql})"+ stripped = (+ f"if(empty({field_sql}), '', {JSON_DROP_KEYS_CLICKHOUSE_NAME}({guarded_sql}, {keys_placeholder}))"+ )if not mask_stored_feature_flags:return strippeddiff --git a/posthog/test/base.py b/posthog/test/base.pyindex e1cdee4d9051..04f11c42827f 100644--- a/posthog/test/base.py+++ b/posthog/test/base.py@@ -326,9 +326,6 @@ def clean_varying_query_parts(query, replace_all_numbers):# remove version suffix from funnel UDFsquery = re.sub(r"aggregate_funnel(_(?:array|cohort))?(_trends)?(_json)?_v\d+", r"aggregate_funnel\1\2\3", query)- # remove version suffix from the restricted-property blob-strip UDF- query = re.sub(r"JSONDropKeys_v\d+", "JSONDropKeys", query)-# replace django cursorsquery = re.sub(r"_django_curs_[0-9sync_]*\"", r'_django_curs_X"', query)