Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-exports): export decimal columns to snowflake (#124023)

Rossdd3c7f017a2a
diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
index 322772b97da4..9342df013747 100644
--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
@@ -282,6 +282,8 @@ SnowflakeTypeName = typing.Literal[
"TINYINT",
"BYTEINT",
"FIXED",
+ # Fixed-point number with a non-zero scale, created as 'NUMBER(precision, scale)'
+ "NUMBER",
# All aliases of the same 64 bit float type
"FLOAT",
"FLOAT4",
@@ -301,17 +303,25 @@ SnowflakeTypeName = typing.Literal[
class SnowflakeType(typing.NamedTuple):
name: SnowflakeTypeName
repeated: bool
+ precision: int | None = None
+ scale: int | None = None
+
+
+SNOWFLAKE_MAX_NUMBER_PRECISION = 38
class SnowflakeDestinationField(typing.NamedTuple):
name: str
type: SnowflakeTypeName
is_nullable: bool
+ precision: int | None = None
+ scale: int | None = None
def data_type_to_snowflake_type(data_type: pa.DataType) -> SnowflakeType:
"""Mapping of `pa.DataType` to corresponding `SnowflakeType`."""
- repeated = False
+ precision: int | None = None
+ scale: int | None = None
if pa.types.is_string(data_type):
snowflake_type_name: SnowflakeTypeName = "STRING"
@@ -332,6 +342,10 @@ def data_type_to_snowflake_type(data_type: pa.DataType) -> SnowflakeType:
elif pa.types.is_floating(data_type):
snowflake_type_name = "FLOAT"
+ elif pa.types.is_decimal(data_type) and data_type.precision <= SNOWFLAKE_MAX_NUMBER_PRECISION:
+ snowflake_type_name = "NUMBER"
+ precision, scale = data_type.precision, data_type.scale
+
elif pa.types.is_boolean(data_type):
snowflake_type_name = "BOOLEAN"
@@ -342,13 +356,12 @@ def data_type_to_snowflake_type(data_type: pa.DataType) -> SnowflakeType:
snowflake_type_name = "TIMESTAMP"
elif pa.types.is_list(data_type):
- repeated = True
- snowflake_type_name = data_type_to_snowflake_type(data_type.value_type).name
+ return data_type_to_snowflake_type(data_type.value_type)._replace(repeated=True)
else:
raise TypeError(f"Unsupported type: '{data_type}'")
- return SnowflakeType(snowflake_type_name, repeated=repeated)
+ return SnowflakeType(snowflake_type_name, repeated=False, precision=precision, scale=scale)
def snowflake_type_to_data_type(type: SnowflakeType) -> pa.DataType:
@@ -370,6 +383,9 @@ def snowflake_type_to_data_type(type: SnowflakeType) -> pa.DataType:
base_type = pa.timestamp("ms", tz="UTC")
case "FLOAT" | "FLOAT4" | "FLOAT8" | "DOUBLE PRECISION" | "REAL":
base_type = pa.float64()
+ case "NUMBER":
+ assert type.precision is not None and type.scale is not None
+ base_type = pa.decimal128(type.precision, type.scale)
case _:
raise ValueError(f"Unsupported type: '{type.name}'")
@@ -401,6 +417,11 @@ class SnowflakeField(Field):
# it. For now, only support arrays of string.
# TODO: Maybe support more arrays?
snowflake_type = SnowflakeType(name="STRING", repeated=True)
+ elif field.type == "NUMBER" or (field.type == "FIXED" and field.scale):
+ # Snowflake reports every 'NUMBER' column as 'FIXED'. A non-zero scale means
+ # the column holds decimals, and reading it as 'INTEGER' would truncate them.
+ # 'NUMBER' itself only comes from `to_destination_field`.
+ snowflake_type = SnowflakeType(name="NUMBER", repeated=False, precision=field.precision, scale=field.scale)
elif field.type == "FIXED":
# This isn't documented as a type. I know it is 'INTEGER', not sure why the
# snowflake library invented 'FIXED'.
@@ -422,8 +443,22 @@ class SnowflakeField(Field):
else:
return self.snowflake_type.name
+ @property
+ def snowflake_type_ddl(self) -> str:
+ if self.snowflake_type_name == "NUMBER":
+ # A bare 'NUMBER' is 'NUMBER(38, 0)', which would round away all decimals.
+ assert self.snowflake_type.precision is not None and self.snowflake_type.scale is not None
+ return f"NUMBER({self.snowflake_type.precision}, {self.snowflake_type.scale})"
+ return self.snowflake_type_name
+
def to_destination_field(self) -> SnowflakeDestinationField:
- return SnowflakeDestinationField(name=self.name, type=self.snowflake_type_name, is_nullable=self.nullable)
+ return SnowflakeDestinationField(
+ name=self.name,
+ type=self.snowflake_type_name,
+ is_nullable=self.nullable,
+ precision=self.snowflake_type.precision,
+ scale=self.snowflake_type.scale,
+ )
def with_new_arrow_type(self, new_type: pa.DataType) -> "SnowflakeField":
return SnowflakeField(self.name, data_type_to_snowflake_type(new_type), new_type, self.nullable)
@@ -933,6 +968,8 @@ class SnowflakeClient:
field_metadata.name,
FIELD_ID_TO_NAME[field_metadata.type_code], # type: ignore[arg-type]
field_metadata.is_nullable,
+ field_metadata.precision,
+ field_metadata.scale,
)
for field_metadata in metadata
if field_metadata.name.lower() in record_batch_field_names
@@ -965,7 +1002,7 @@ class SnowflakeClient:
won't fail if the table already exists.
"""
field_ddl = ", ".join(
- f"{_quote_snowflake_identifier(field.name)} {field.snowflake_type_name}" for field in table
+ f"{_quote_snowflake_identifier(field.name)} {field.snowflake_type_ddl}" for field in table
)
if_not_exists = ""
diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py
index cdc6c58697b0..013be2ec6513 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py
@@ -157,6 +157,7 @@ async def test_table(snowflake_client, database, schema, table_name):
fields=(
SnowflakeField("id", SnowflakeType("INTEGER", False), pa.int64(), True),
SnowflakeField("text", SnowflakeType("TEXT", False), pa.string(), True),
+ SnowflakeField("amount", SnowflakeType("NUMBER", False, 38, 2), pa.decimal128(38, 2), True),
),
parents=(database, schema),
)
diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_field.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_field.py
new file mode 100644
index 000000000000..da3068ddda4b
--- /dev/null
+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_field.py
@@ -0,0 +1,15 @@
+import pyarrow as pa
+
+from products.batch_exports.backend.temporal.destinations.snowflake_batch_export import (
+ SnowflakeDestinationField,
+ SnowflakeField,
+)
+
+
+def test_decimal_field_maps_to_number_and_reads_back_as_decimal() -> None:
+ field = SnowflakeField.from_arrow_field(pa.field("amount", pa.decimal128(38, 2)))
+
+ assert field.snowflake_type_ddl == "NUMBER(38, 2)"
+ # Snowflake reports a 'NUMBER(38, 2)' column as 'FIXED' with precision 38 and scale 2.
+ assert SnowflakeField.from_destination_field(SnowflakeDestinationField("amount", "FIXED", True, 38, 2)) == field
+ assert SnowflakeField.from_destination_field(field.to_destination_field()) == field
diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py
index 1162ab9fb0fe..7379e502f195 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py
@@ -100,6 +100,8 @@ class FakeMetadata(typing.NamedTuple):
name: str
type_code: int
is_nullable: bool
+ precision: int | None = None
+ scale: int | None = None
class FakeSnowflakeCursor:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/snowflake.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/snowflake.py
index 448a6c75be94..64a2fb7f54b9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/snowflake.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/snowflake.py
@@ -8,11 +8,11 @@ their own typed errors.
Two pieces of batch exports are deliberately not reused:
-- The Arrow to Snowflake type mapping behind `SnowflakeField.from_arrow_field`. Its vocabulary
- has no `DATE`, `TIME` or `NUMBER(p, s)` and it raises on anything else, which is fine for
- events and persons but would silently turn a source date column into a timestamp and a
- decimal into a float. The DDL here keeps those types. The table handed to the shared stage
- helpers carries the Arrow types instead, which is all those helpers read.
+- The Arrow to Snowflake type mapping behind `SnowflakeField.from_arrow_field`. It has no
+ `TIME`, writes every timestamp as `TIMESTAMP` whatever its time zone, and raises on structs
+ and maps. That is fine for events and persons but not for an arbitrary source table. The DDL
+ here keeps those types. The table handed to the shared stage helpers carries the Arrow types
+ instead, which is all those helpers read.
- `merge_into_final_from_stage`, which needs a version key to decide what to update. A synced
source table has no monotonic column, so the merge below matches on primary keys alone.
@@ -202,8 +202,8 @@ def _stage_table(table: str, schema: pa.Schema, stage_prefix: str) -> SnowflakeT
`remove_internal_stage_files`, `put_file_to_snowflake_table_stage` and
`copy_loaded_files_to_snowflake_table` read a table's name, its stage prefix, and each
field's name and Arrow type. They never read the Snowflake type, which is why the column
- types this writer maps (`DATE`, `TIME`, `NUMBER(p, s)`) can stay out of a vocabulary that
- has no room for them, and why the placeholder passed in their place is safe.
+ types this writer maps can differ from batch exports' own mapping, and why the placeholder
+ passed in their place is safe.
"""
_assert_safe_identifier(table, "table name")
fields = [