Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-exports): load malformed JSON values as NULL in Snowflake (#129954)

Ross4a22dd0b1496
diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
index a59923331f4d..86c1f71b479d 100644
--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
@@ -654,10 +654,29 @@ def _is_connect_error_retryable(err: Exception) -> bool:
return isinstance(err, OperationalError) and err.errno in (ER_FAILED_TO_CONNECT_TO_DB, ER_RETRYABLE_CODE)
+# These columns hold `$set` and `$set_once` and can include any text sent by a client. PARSE_JSON
+# fails the whole COPY on one malformed value, and unfortunately COPY transformations do not support
+# TRY_PARSE_JSON (at least according to the Snowflake docs), so we use CHECK_JSON as a guard
+# instead. Other JSON columns do not use this guard: we currently accept duplicate keys in JSON
+# objects and CHECK_JSON has no option to support this like PARSE_JSON does with the 'd' option.
+COLUMNS_WITH_UNVALIDATED_JSON = frozenset({"people_set", "people_set_once"})
+
+
def _quote_snowflake_identifier(identifier: str) -> str:
return '"' + identifier.replace('"', '""') + '"'
+def _build_copy_select_expression(field: Field) -> str:
+ """Return the expression that a COPY INTO uses to load `field` from a staged Parquet file."""
+ column = f"$1:{_quote_snowflake_identifier(field.name)}"
+ name = field.name.lower()
+ if field.data_type != JsonType() or name == "elements":
+ return column
+ if name in COLUMNS_WITH_UNVALIDATED_JSON:
+ return f"IFF(CHECK_JSON({column}) IS NULL, PARSE_JSON({column}, 'd'), NULL)"
+ return f"PARSE_JSON({column}, 'd')"
+
+
class SnowflakeClient:
"""Snowflake connection client used in batch exports."""
@@ -1192,12 +1211,7 @@ class SnowflakeClient:
SnowflakeInsufficientPrivilegesError: If the configured Snowflake role lacks required privileges to
perform this operation.
"""
- select_fields = ", ".join(
- f"PARSE_JSON($1:{_quote_snowflake_identifier(field.name)}, 'd')"
- if field.data_type == JsonType() and field.name.lower() != "elements"
- else f"$1:{_quote_snowflake_identifier(field.name)}"
- for field in table
- )
+ select_fields = ", ".join(_build_copy_select_expression(field) for field in table)
query = f"""
COPY INTO "{table.name}" ({", ".join(_quote_snowflake_identifier(field.name) for field in table)})
diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py
index 8ce21cd57d9b..66686f00cc90 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py
@@ -5,6 +5,7 @@ Note: This module uses a real Snowflake connection.
"""
import os
+import json
import uuid
import typing as t
import datetime as dt
@@ -784,3 +785,54 @@ async def test_insert_into_snowflake_activity_handles_extra_columns_in_destinati
extra_fields={"extra_column": "test"},
min_ingested_timestamp=min_ingested_timestamp,
)
+
+
+async def test_insert_into_snowflake_activity_loads_malformed_people_set_as_null(
+ clickhouse_client,
+ activity_environment,
+ snowflake_cursor,
+ snowflake_activity_inputs,
+ data_interval_start,
+ data_interval_end,
+ ateam,
+):
+ for event_name, properties in (
+ ("malformed", {"$set": "not valid json", "$set_once": "{also not valid"}),
+ ("well-formed", {"$set": '{"plan": "free"}', "$set_once": '{"plan": "paid"}'}),
+ ):
+ await generate_test_events_in_clickhouse(
+ client=clickhouse_client,
+ team_id=ateam.pk,
+ start_time=data_interval_start,
+ end_time=data_interval_end,
+ count=1,
+ count_outside_range=0,
+ count_other_team=0,
+ event_name=event_name,
+ properties=properties,
+ )
+
+ table_name = f"test_insert_activity_malformed_people_set_{ateam.pk}"
+ result = await _run_activity(
+ activity_environment=activity_environment,
+ snowflake_cursor=snowflake_cursor,
+ clickhouse_client=clickhouse_client,
+ snowflake_activity_inputs=snowflake_activity_inputs,
+ team=ateam,
+ data_interval_start=data_interval_start,
+ data_interval_end=data_interval_end,
+ table_name=table_name,
+ batch_export_model=BatchExportEventsModel(),
+ assert_clickhouse_records=False,
+ )
+
+ assert isinstance(result, BatchExportResult)
+ assert result.error is None
+
+ snowflake_cursor.execute(f'SELECT "event", "people_set", "people_set_once" FROM "{table_name}" ORDER BY "event"')
+ malformed, well_formed = snowflake_cursor.fetchall()
+
+ assert malformed == ("malformed", None, None)
+ assert well_formed[0] == "well-formed"
+ assert json.loads(well_formed[1]) == {"plan": "free"}
+ assert json.loads(well_formed[2]) == {"plan": "paid"}
diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py
index b8db18504549..f5b6dac62721 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py
@@ -56,6 +56,7 @@ async def test_staging_quotes_column_names(create: bool) -> None:
(
SnowflakeField('key"name', SnowflakeType("INT", False), pa.int64(), False),
SnowflakeField('json"data', SnowflakeType("VARIANT", False), JsonType(), True),
+ SnowflakeField("people_set", SnowflakeType("VARIANT", False), JsonType(), True),
),
)
client = SnowflakeClient(
@@ -69,7 +70,13 @@ async def test_staging_quotes_column_names(create: bool) -> None:
query = " ".join(execute.call_args.args[0].split())
if create:
- assert 'CREATE TABLE IF NOT EXISTS "stage" ( "key""name" INT, "json""data" VARIANT )' in query
+ assert (
+ 'CREATE TABLE IF NOT EXISTS "stage" ( "key""name" INT, "json""data" VARIANT, "people_set" VARIANT )'
+ in query
+ )
else:
- assert 'COPY INTO "stage" ("key""name", "json""data")' in query
- assert 'SELECT $1:"key""name", PARSE_JSON($1:"json""data", \'d\')' in query
+ assert 'COPY INTO "stage" ("key""name", "json""data", "people_set")' in query
+ assert (
+ 'SELECT $1:"key""name", PARSE_JSON($1:"json""data", \'d\'), '
+ 'IFF(CHECK_JSON($1:"people_set") IS NULL, PARSE_JSON($1:"people_set", \'d\'), NULL)'
+ ) in query