:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(batch-exports): export decimal columns to snowflake (#124023)
Rossdd3c7f017a2a
diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.pyindex 322772b97da4..9342df013747 100644--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py@@ -282,6 +282,8 @@ SnowflakeTypeName = typing.Literal["TINYINT","BYTEINT","FIXED",+ # Fixed-point number with a non-zero scale, created as 'NUMBER(precision, scale)'+ "NUMBER",# All aliases of the same 64 bit float type"FLOAT","FLOAT4",@@ -301,17 +303,25 @@ SnowflakeTypeName = typing.Literal[class SnowflakeType(typing.NamedTuple):name: SnowflakeTypeNamerepeated: bool+ precision: int | None = None+ scale: int | None = None+++SNOWFLAKE_MAX_NUMBER_PRECISION = 38class SnowflakeDestinationField(typing.NamedTuple):name: strtype: SnowflakeTypeNameis_nullable: bool+ precision: int | None = None+ scale: int | None = Nonedef data_type_to_snowflake_type(data_type: pa.DataType) -> SnowflakeType:"""Mapping of `pa.DataType` to corresponding `SnowflakeType`."""- repeated = False+ precision: int | None = None+ scale: int | None = Noneif pa.types.is_string(data_type):snowflake_type_name: SnowflakeTypeName = "STRING"@@ -332,6 +342,10 @@ def data_type_to_snowflake_type(data_type: pa.DataType) -> SnowflakeType:elif pa.types.is_floating(data_type):snowflake_type_name = "FLOAT"+ elif pa.types.is_decimal(data_type) and data_type.precision <= SNOWFLAKE_MAX_NUMBER_PRECISION:+ snowflake_type_name = "NUMBER"+ precision, scale = data_type.precision, data_type.scale+elif pa.types.is_boolean(data_type):snowflake_type_name = "BOOLEAN"@@ -342,13 +356,12 @@ def data_type_to_snowflake_type(data_type: pa.DataType) -> SnowflakeType:snowflake_type_name = "TIMESTAMP"elif pa.types.is_list(data_type):- repeated = True- snowflake_type_name = data_type_to_snowflake_type(data_type.value_type).name+ return data_type_to_snowflake_type(data_type.value_type)._replace(repeated=True)else:raise TypeError(f"Unsupported type: '{data_type}'")- return SnowflakeType(snowflake_type_name, repeated=repeated)+ return SnowflakeType(snowflake_type_name, repeated=False, precision=precision, scale=scale)def snowflake_type_to_data_type(type: SnowflakeType) -> pa.DataType:@@ -370,6 +383,9 @@ def snowflake_type_to_data_type(type: SnowflakeType) -> pa.DataType:base_type = pa.timestamp("ms", tz="UTC")case "FLOAT" | "FLOAT4" | "FLOAT8" | "DOUBLE PRECISION" | "REAL":base_type = pa.float64()+ case "NUMBER":+ assert type.precision is not None and type.scale is not None+ base_type = pa.decimal128(type.precision, type.scale)case _:raise ValueError(f"Unsupported type: '{type.name}'")@@ -401,6 +417,11 @@ class SnowflakeField(Field):# it. For now, only support arrays of string.# TODO: Maybe support more arrays?snowflake_type = SnowflakeType(name="STRING", repeated=True)+ elif field.type == "NUMBER" or (field.type == "FIXED" and field.scale):+ # Snowflake reports every 'NUMBER' column as 'FIXED'. A non-zero scale means+ # the column holds decimals, and reading it as 'INTEGER' would truncate them.+ # 'NUMBER' itself only comes from `to_destination_field`.+ snowflake_type = SnowflakeType(name="NUMBER", repeated=False, precision=field.precision, scale=field.scale)elif field.type == "FIXED":# This isn't documented as a type. I know it is 'INTEGER', not sure why the# snowflake library invented 'FIXED'.@@ -422,8 +443,22 @@ class SnowflakeField(Field):else:return self.snowflake_type.name+ @property+ def snowflake_type_ddl(self) -> str:+ if self.snowflake_type_name == "NUMBER":+ # A bare 'NUMBER' is 'NUMBER(38, 0)', which would round away all decimals.+ assert self.snowflake_type.precision is not None and self.snowflake_type.scale is not None+ return f"NUMBER({self.snowflake_type.precision}, {self.snowflake_type.scale})"+ return self.snowflake_type_name+def to_destination_field(self) -> SnowflakeDestinationField:- return SnowflakeDestinationField(name=self.name, type=self.snowflake_type_name, is_nullable=self.nullable)+ return SnowflakeDestinationField(+ name=self.name,+ type=self.snowflake_type_name,+ is_nullable=self.nullable,+ precision=self.snowflake_type.precision,+ scale=self.snowflake_type.scale,+ )def with_new_arrow_type(self, new_type: pa.DataType) -> "SnowflakeField":return SnowflakeField(self.name, data_type_to_snowflake_type(new_type), new_type, self.nullable)@@ -933,6 +968,8 @@ class SnowflakeClient:field_metadata.name,FIELD_ID_TO_NAME[field_metadata.type_code], # type: ignore[arg-type]field_metadata.is_nullable,+ field_metadata.precision,+ field_metadata.scale,)for field_metadata in metadataif field_metadata.name.lower() in record_batch_field_names@@ -965,7 +1002,7 @@ class SnowflakeClient:won't fail if the table already exists."""field_ddl = ", ".join(- f"{_quote_snowflake_identifier(field.name)} {field.snowflake_type_name}" for field in table+ f"{_quote_snowflake_identifier(field.name)} {field.snowflake_type_ddl}" for field in table)if_not_exists = ""diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.pyindex cdc6c58697b0..013be2ec6513 100644--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py@@ -157,6 +157,7 @@ async def test_table(snowflake_client, database, schema, table_name):fields=(SnowflakeField("id", SnowflakeType("INTEGER", False), pa.int64(), True),SnowflakeField("text", SnowflakeType("TEXT", False), pa.string(), True),+ SnowflakeField("amount", SnowflakeType("NUMBER", False, 38, 2), pa.decimal128(38, 2), True),),parents=(database, schema),)diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_field.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_field.pynew file mode 100644index 000000000000..da3068ddda4b--- /dev/null+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_field.py@@ -0,0 +1,15 @@+import pyarrow as pa++from products.batch_exports.backend.temporal.destinations.snowflake_batch_export import (+ SnowflakeDestinationField,+ SnowflakeField,+)+++def test_decimal_field_maps_to_number_and_reads_back_as_decimal() -> None:+ field = SnowflakeField.from_arrow_field(pa.field("amount", pa.decimal128(38, 2)))++ assert field.snowflake_type_ddl == "NUMBER(38, 2)"+ # Snowflake reports a 'NUMBER(38, 2)' column as 'FIXED' with precision 38 and scale 2.+ assert SnowflakeField.from_destination_field(SnowflakeDestinationField("amount", "FIXED", True, 38, 2)) == field+ assert SnowflakeField.from_destination_field(field.to_destination_field()) == fielddiff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.pyindex 1162ab9fb0fe..7379e502f195 100644--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py@@ -100,6 +100,8 @@ class FakeMetadata(typing.NamedTuple):name: strtype_code: intis_nullable: bool+ precision: int | None = None+ scale: int | None = Noneclass FakeSnowflakeCursor:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/snowflake.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/snowflake.pyindex 448a6c75be94..64a2fb7f54b9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/snowflake.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/snowflake.py@@ -8,11 +8,11 @@ their own typed errors.Two pieces of batch exports are deliberately not reused:-- The Arrow to Snowflake type mapping behind `SnowflakeField.from_arrow_field`. Its vocabulary- has no `DATE`, `TIME` or `NUMBER(p, s)` and it raises on anything else, which is fine for- events and persons but would silently turn a source date column into a timestamp and a- decimal into a float. The DDL here keeps those types. The table handed to the shared stage- helpers carries the Arrow types instead, which is all those helpers read.+- The Arrow to Snowflake type mapping behind `SnowflakeField.from_arrow_field`. It has no+ `TIME`, writes every timestamp as `TIMESTAMP` whatever its time zone, and raises on structs+ and maps. That is fine for events and persons but not for an arbitrary source table. The DDL+ here keeps those types. The table handed to the shared stage helpers carries the Arrow types+ instead, which is all those helpers read.- `merge_into_final_from_stage`, which needs a version key to decide what to update. A syncedsource table has no monotonic column, so the merge below matches on primary keys alone.@@ -202,8 +202,8 @@ def _stage_table(table: str, schema: pa.Schema, stage_prefix: str) -> SnowflakeT`remove_internal_stage_files`, `put_file_to_snowflake_table_stage` and`copy_loaded_files_to_snowflake_table` read a table's name, its stage prefix, and eachfield's name and Arrow type. They never read the Snowflake type, which is why the column- types this writer maps (`DATE`, `TIME`, `NUMBER(p, s)`) can stay out of a vocabulary that- has no room for them, and why the placeholder passed in their place is safe.+ types this writer maps can differ from batch exports' own mapping, and why the placeholder+ passed in their place is safe."""_assert_safe_identifier(table, "table name")fields = [