Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): strip restricted property keys with the pooled udf (#120479)

Rory Shanksf03f94bf040c
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.md
index 64777bc540b0..90f981b8119c 100644
--- a/posthog/hogql/ACCESS_CONTROL.md
+++ b/posthog/hogql/ACCESS_CONTROL.md
@@ -185,7 +185,7 @@ Unlike levels 1 and 2, restricted properties do not error.
They're masked when the query is printed to ClickHouse SQL, so a restricted read is indistinguishable from a property that was never set (anti-enumeration):
- **Explicit reads** (`properties.email`) are replaced with `NULL`, and the resolver refuses to back them with a materialized column — `ClickHousePropertyResolver` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.
-- **Whole-blob reads** (`SELECT properties` or `SELECT *`) have the restricted keys stripped from the returned JSON via `JSONDropKeys(...)` — `ClickHousePrinter._maybe_apply_json_drop_keys()` in `posthog/hogql/printer/clickhouse.py`.
+- **Whole-blob reads** (`SELECT properties` or `SELECT *`) have the restricted keys stripped from the returned JSON via `JSONDropKeysPool(...)` — `ClickHousePrinter._maybe_apply_json_drop_keys()` in `posthog/hogql/printer/clickhouse.py`.
Group restrictions retain their group type index, so a same-named property on another group type stays readable. The masking also applies to the Postgres-backed `system.groups.group_properties` field.
diff --git a/posthog/hogql/functions/udfs.py b/posthog/hogql/functions/udfs.py
index 368509dea093..3e80f42ca13c 100644
--- a/posthog/hogql/functions/udfs.py
+++ b/posthog/hogql/functions/udfs.py
@@ -28,9 +28,11 @@ UDFS: dict[str, HogQLFunctionMeta] = {
),
}
-# JSONDropKeys is an executable UDF like the funnel UDFs, but it is printer-internal (restricted-property blob
-# stripping), not HogQL-exposed, so it lives outside UDFS. Its deployed name is versioned the same way.
-JSON_DROP_KEYS_CLICKHOUSE_NAME = "JSONDropKeys"
+# JSONDropKeysPool strips restricted keys from a JSON blob. It is printer-internal, not HogQL-exposed, so it lives
+# outside UDFS. It is an executable_pool UDF in UNVERSIONED_FUNCTIONS, so its deployed name has no version suffix.
+# Do not use JSONDropKeys here: it takes the keys as a command-line parameter, which forces the `executable` type,
+# and ClickHouse then starts a new process for every block.
+JSON_DROP_KEYS_CLICKHOUSE_NAME = "JSONDropKeysPool"
# We want CI to fail if there is a breaking change and the version hasn't been incremented
if is_cloud() or is_ci():
@@ -38,4 +40,3 @@ if is_cloud() or is_ci():
for v in UDFS.values():
v.clickhouse_name = augment_function_name(v.clickhouse_name)
- JSON_DROP_KEYS_CLICKHOUSE_NAME = augment_function_name(JSON_DROP_KEYS_CLICKHOUSE_NAME)
diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.py
index 5acd6bd92990..92bb29e3e75d 100644
--- a/posthog/hogql/printer/clickhouse.py
+++ b/posthog/hogql/printer/clickhouse.py
@@ -683,13 +683,13 @@ class ClickHousePrinter(BasePrinter):
def _maybe_apply_json_drop_keys(self, type: ast.FieldType, field_sql: str, mask_feature_flags: bool = False) -> str:
"""
- Wraps a StringJSONDatabaseField in JSONDropKeys() to strip restricted property keys
+ Wraps a StringJSONDatabaseField in JSONDropKeysPool() to strip restricted property keys
when the raw JSON blob is selected directly (e.g., `SELECT properties FROM events`).
`mask_feature_flags` is for an events_json `properties` column printed as stored rather than as the rebuilt
document, which happens under `toJSONString(if(..., properties, ...))`. That document holds the flags in the
- `$feature_flags` map, where JSONDropKeys cannot reach a restricted `$feature/<key>`, so the map is dropped and
- merged back without the restricted flags.
+ `$feature_flags` map, where JSONDropKeysPool cannot reach a restricted `$feature/<key>`, so the map is dropped
+ and merged back without the restricted flags.
"""
if not self.context.restricted_properties:
return field_sql
@@ -701,7 +701,7 @@ class ClickHousePrinter(BasePrinter):
# Use the resolved DB column name, not ``type.name``. With column-alias table syntax
# (``FROM events AS e(uuid, event, ..., p)``) the AST field name is the alias (``p``),
# but ClickHouse resolves it back to the original column. Comparing ``type.name`` here
- # would incorrectly skip JSONDropKeys wrapping for the aliased ``properties`` column.
+ # would incorrectly skip JSONDropKeysPool wrapping for the aliased ``properties`` column.
# ``person_properties`` is the underlying DB column for ``EventsPersonSubTable.properties``
# (PoE mode); it is also a JSON blob that must be stripped of restricted person-property keys.
if resolved_field.name not in RESTRICTABLE_JSON_BLOB_COLUMNS:
@@ -733,13 +733,19 @@ class ClickHousePrinter(BasePrinter):
}
keys_placeholder = self.context.add_sensitive_value(sorted(keys_to_drop))
- stripped = f"{JSON_DROP_KEYS_CLICKHOUSE_NAME}({keys_placeholder})({field_sql})"
- if not is_events_json_document:
- # A String JSON column holds '' when its producer omits the value.
- # The v12 json_drop_keys_udf binary fails to parse ''. Cloud and CI call it as JSONDropKeys_v12.
- # ClickHouse short-circuits `if`. The UDF therefore receives only the rows that are not empty.
+ if is_events_json_document:
# A native events document is a JSON column or a document that concat rebuilds, so it is never ''.
- stripped = f"if(empty({field_sql}), '', {stripped})"
+ # The UDF reads RowBinary, which encodes a JSON column differently from a String. Without toString the
+ # worker misreads the row and waits for more input until ClickHouse times out the pipe.
+ stripped = f"{JSON_DROP_KEYS_CLICKHOUSE_NAME}(toString({field_sql}), {keys_placeholder})"
+ else:
+ # A String JSON column holds '' when its producer omits the value, and the UDF fails to parse ''.
+ # ClickHouse can evaluate both branches of `if`, so the UDF gets '{}' in place of '' and the outer
+ # `if` discards that result.
+ guarded_sql = f"if(empty({field_sql}), '{{}}', {field_sql})"
+ stripped = (
+ f"if(empty({field_sql}), '', {JSON_DROP_KEYS_CLICKHOUSE_NAME}({guarded_sql}, {keys_placeholder}))"
+ )
if not mask_stored_feature_flags:
return stripped
diff --git a/posthog/test/base.py b/posthog/test/base.py
index e1cdee4d9051..04f11c42827f 100644
--- a/posthog/test/base.py
+++ b/posthog/test/base.py
@@ -326,9 +326,6 @@ def clean_varying_query_parts(query, replace_all_numbers):
# remove version suffix from funnel UDFs
query = re.sub(r"aggregate_funnel(_(?:array|cohort))?(_trends)?(_json)?_v\d+", r"aggregate_funnel\1\2\3", query)
- # remove version suffix from the restricted-property blob-strip UDF
- query = re.sub(r"JSONDropKeys_v\d+", "JSONDropKeys", query)
-
# replace django cursors
query = re.sub(r"_django_curs_[0-9sync_]*\"", r'_django_curs_X"', query)