:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(batch-exports): load malformed JSON values as NULL in Snowflake (#129954)
Ross4a22dd0b1496
diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.pyindex a59923331f4d..86c1f71b479d 100644--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py@@ -654,10 +654,29 @@ def _is_connect_error_retryable(err: Exception) -> bool:return isinstance(err, OperationalError) and err.errno in (ER_FAILED_TO_CONNECT_TO_DB, ER_RETRYABLE_CODE)+# These columns hold `$set` and `$set_once` and can include any text sent by a client. PARSE_JSON+# fails the whole COPY on one malformed value, and unfortunately COPY transformations do not support+# TRY_PARSE_JSON (at least according to the Snowflake docs), so we use CHECK_JSON as a guard+# instead. Other JSON columns do not use this guard: we currently accept duplicate keys in JSON+# objects and CHECK_JSON has no option to support this like PARSE_JSON does with the 'd' option.+COLUMNS_WITH_UNVALIDATED_JSON = frozenset({"people_set", "people_set_once"})++def _quote_snowflake_identifier(identifier: str) -> str:return '"' + identifier.replace('"', '""') + '"'+def _build_copy_select_expression(field: Field) -> str:+ """Return the expression that a COPY INTO uses to load `field` from a staged Parquet file."""+ column = f"$1:{_quote_snowflake_identifier(field.name)}"+ name = field.name.lower()+ if field.data_type != JsonType() or name == "elements":+ return column+ if name in COLUMNS_WITH_UNVALIDATED_JSON:+ return f"IFF(CHECK_JSON({column}) IS NULL, PARSE_JSON({column}, 'd'), NULL)"+ return f"PARSE_JSON({column}, 'd')"++class SnowflakeClient:"""Snowflake connection client used in batch exports."""@@ -1192,12 +1211,7 @@ class SnowflakeClient:SnowflakeInsufficientPrivilegesError: If the configured Snowflake role lacks required privileges toperform this operation."""- select_fields = ", ".join(- f"PARSE_JSON($1:{_quote_snowflake_identifier(field.name)}, 'd')"- if field.data_type == JsonType() and field.name.lower() != "elements"- else f"$1:{_quote_snowflake_identifier(field.name)}"- for field in table- )+ select_fields = ", ".join(_build_copy_select_expression(field) for field in table)query = f"""COPY INTO "{table.name}" ({", ".join(_quote_snowflake_identifier(field.name) for field in table)})diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.pyindex 8ce21cd57d9b..66686f00cc90 100644--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_activity_e2e.py@@ -5,6 +5,7 @@ Note: This module uses a real Snowflake connection."""import os+import jsonimport uuidimport typing as timport datetime as dt@@ -784,3 +785,54 @@ async def test_insert_into_snowflake_activity_handles_extra_columns_in_destinatiextra_fields={"extra_column": "test"},min_ingested_timestamp=min_ingested_timestamp,)+++async def test_insert_into_snowflake_activity_loads_malformed_people_set_as_null(+ clickhouse_client,+ activity_environment,+ snowflake_cursor,+ snowflake_activity_inputs,+ data_interval_start,+ data_interval_end,+ ateam,+):+ for event_name, properties in (+ ("malformed", {"$set": "not valid json", "$set_once": "{also not valid"}),+ ("well-formed", {"$set": '{"plan": "free"}', "$set_once": '{"plan": "paid"}'}),+ ):+ await generate_test_events_in_clickhouse(+ client=clickhouse_client,+ team_id=ateam.pk,+ start_time=data_interval_start,+ end_time=data_interval_end,+ count=1,+ count_outside_range=0,+ count_other_team=0,+ event_name=event_name,+ properties=properties,+ )++ table_name = f"test_insert_activity_malformed_people_set_{ateam.pk}"+ result = await _run_activity(+ activity_environment=activity_environment,+ snowflake_cursor=snowflake_cursor,+ clickhouse_client=clickhouse_client,+ snowflake_activity_inputs=snowflake_activity_inputs,+ team=ateam,+ data_interval_start=data_interval_start,+ data_interval_end=data_interval_end,+ table_name=table_name,+ batch_export_model=BatchExportEventsModel(),+ assert_clickhouse_records=False,+ )++ assert isinstance(result, BatchExportResult)+ assert result.error is None++ snowflake_cursor.execute(f'SELECT "event", "people_set", "people_set_once" FROM "{table_name}" ORDER BY "event"')+ malformed, well_formed = snowflake_cursor.fetchall()++ assert malformed == ("malformed", None, None)+ assert well_formed[0] == "well-formed"+ assert json.loads(well_formed[1]) == {"plan": "free"}+ assert json.loads(well_formed[2]) == {"plan": "paid"}diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.pyindex b8db18504549..f5b6dac62721 100644--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_merge.py@@ -56,6 +56,7 @@ async def test_staging_quotes_column_names(create: bool) -> None:(SnowflakeField('key"name', SnowflakeType("INT", False), pa.int64(), False),SnowflakeField('json"data', SnowflakeType("VARIANT", False), JsonType(), True),+ SnowflakeField("people_set", SnowflakeType("VARIANT", False), JsonType(), True),),)client = SnowflakeClient(@@ -69,7 +70,13 @@ async def test_staging_quotes_column_names(create: bool) -> None:query = " ".join(execute.call_args.args[0].split())if create:- assert 'CREATE TABLE IF NOT EXISTS "stage" ( "key""name" INT, "json""data" VARIANT )' in query+ assert (+ 'CREATE TABLE IF NOT EXISTS "stage" ( "key""name" INT, "json""data" VARIANT, "people_set" VARIANT )'+ in query+ )else:- assert 'COPY INTO "stage" ("key""name", "json""data")' in query- assert 'SELECT $1:"key""name", PARSE_JSON($1:"json""data", \'d\')' in query+ assert 'COPY INTO "stage" ("key""name", "json""data", "people_set")' in query+ assert (+ 'SELECT $1:"key""name", PARSE_JSON($1:"json""data", \'d\'), '+ 'IFF(CHECK_JSON($1:"people_set") IS NULL, PARSE_JSON($1:"people_set", \'d\'), NULL)'+ ) in query