Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-exports): log unsupported column types while retrying (#121670)

Phill5c9c06c76b7f
diff --git a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
index 07fbb41588e2..50ecaf739ad9 100644
--- a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
@@ -63,6 +63,7 @@ from products.batch_exports.backend.temporal.batch_exports import (
is_over_billing_limit_error,
start_batch_export_run,
)
+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingError
from products.batch_exports.backend.temporal.pipeline.consumer import Consumer
from products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stage
from products.batch_exports.backend.temporal.pipeline.producer import Producer
@@ -81,6 +82,7 @@ from products.batch_exports.backend.temporal.queue import (
)
from products.batch_exports.backend.temporal.utils import (
JsonType,
+ handle_column_type_mapping_errors,
handle_non_retryable_errors,
make_retryable_with_exponential_backoff,
)
@@ -272,7 +274,10 @@ class BigQueryField(Field):
@classmethod
def from_arrow_field(cls, field: pa.Field) -> typing.Self:
- type = data_type_to_bigquery_type(field.type)
+ try:
+ type = data_type_to_bigquery_type(field.type)
+ except ValueError as err:
+ raise ColumnTypeMappingError("BigQuery", field.name, field.type) from err
return cls(field.name, type, nullable=field.nullable)
@classmethod
@@ -1577,14 +1582,15 @@ async def insert_into_bigquery_activity_from_stage(inputs: BigQueryInsertInputs)
json_fields = set()
requires_merge = model.incremental_mode == IncrementalMode.MERGE
- target_table = BigQueryTable.from_arrow_schema(
- record_batch_schema,
- table_id=inputs.table_id,
- project_id=project_id,
- dataset_id=inputs.dataset_id,
- primary_key=model.primary_key if requires_merge else (),
- version_key=model.version_key if requires_merge else (),
- )
+ with handle_column_type_mapping_errors(model):
+ target_table = BigQueryTable.from_arrow_schema(
+ record_batch_schema,
+ table_id=inputs.table_id,
+ project_id=project_id,
+ dataset_id=inputs.dataset_id,
+ primary_key=model.primary_key if requires_merge else (),
+ version_key=model.version_key if requires_merge else (),
+ )
data_interval_end_str = dt.datetime.fromisoformat(inputs.data_interval_end).strftime("%Y-%m-%d_%H-%M-%S")
attempt = activity.info().attempt
diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
index 557bf5323e99..d5291ab64b17 100644
--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
@@ -47,6 +47,7 @@ from products.batch_exports.backend.temporal.batch_exports import (
is_over_billing_limit_error,
start_batch_export_run,
)
+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingError
from products.batch_exports.backend.temporal.pipeline.consumer import Consumer, run_consumer_from_stage
from products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stage
from products.batch_exports.backend.temporal.pipeline.producer import Producer
@@ -58,6 +59,7 @@ from products.batch_exports.backend.temporal.pipeline.types import BatchExportRe
from products.batch_exports.backend.temporal.queue import RecordBatchQueue, wait_for_schema_or_producer
from products.batch_exports.backend.temporal.utils import (
JsonType,
+ handle_column_type_mapping_errors,
handle_non_retryable_errors,
make_retryable_with_exponential_backoff,
)
@@ -1037,7 +1039,7 @@ def _get_databricks_fields_from_record_schema(
is_variant = pa_field.name in known_variant_columns
databricks_type = _get_databricks_field_type(pa_field.type, is_variant)
if databricks_type is None:
- raise TypeError(f"Unsupported type in field '{name}': '{databricks_type}'")
+ raise ColumnTypeMappingError("Databricks", name, pa_field.type)
databricks_schema.append((name, databricks_type))
return databricks_schema
@@ -1110,10 +1112,11 @@ def _get_databricks_table_settings(
if staged_fields:
table_fields = staged_fields
else:
- table_fields = _get_databricks_fields_from_record_schema(
- record_batch_schema,
- known_variant_columns=known_variant_columns,
- )
+ with handle_column_type_mapping_errors(model):
+ table_fields = _get_databricks_fields_from_record_schema(
+ record_batch_schema,
+ known_variant_columns=known_variant_columns,
+ )
return TableSettings(table_fields, record_batch_schema, known_variant_columns)
diff --git a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
index 67d9e52be2f3..ed059fc75ab3 100644
--- a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
@@ -53,6 +53,7 @@ from products.batch_exports.backend.temporal.batch_exports import (
start_batch_export_run,
)
from products.batch_exports.backend.temporal.destinations.utils import get_query_timeout
+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingError
from products.batch_exports.backend.temporal.pipeline.consumer import Consumer, run_consumer_from_stage
from products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stage
from products.batch_exports.backend.temporal.pipeline.producer import Producer
@@ -61,6 +62,7 @@ from products.batch_exports.backend.temporal.pipeline.types import BatchExportRe
from products.batch_exports.backend.temporal.queue import RecordBatchQueue, wait_for_schema_or_producer
from products.batch_exports.backend.temporal.utils import (
JsonType,
+ handle_column_type_mapping_errors,
handle_non_retryable_errors,
make_retryable_with_exponential_backoff,
)
@@ -838,7 +840,7 @@ def get_postgres_fields_from_record_schema(
pg_type = "TEXT[]"
else:
- raise TypeError(f"Unsupported type in field '{name}': '{pa_field.type}'")
+ raise ColumnTypeMappingError("PostgreSQL", name, pa_field.type)
pg_schema.append((name, pg_type))
@@ -934,10 +936,11 @@ def _get_table_fields(
# A retry can consume files staged before a new default column was added.
return [field for field in table_fields if field[0] in record_batch_schema.names]
else:
- return get_postgres_fields_from_record_schema(
- record_batch_schema,
- known_json_columns=["properties", "set", "set_once", "person_properties"],
- )
+ with handle_column_type_mapping_errors(model):
+ return get_postgres_fields_from_record_schema(
+ record_batch_schema,
+ known_json_columns=["properties", "set", "set_once", "person_properties"],
+ )
class PostgreSQLIntegrationNotFoundError(Exception):
diff --git a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
index a46287eda88d..a1aceee13e81 100644
--- a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
@@ -70,7 +70,7 @@ from products.batch_exports.backend.temporal.destinations.s3_batch_export import
s3_client,
)
from products.batch_exports.backend.temporal.destinations.utils import get_absolute_key_prefix
-from products.batch_exports.backend.temporal.errors import MissingRequiredInputsError
+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingError, MissingRequiredInputsError
from products.batch_exports.backend.temporal.pipeline.consumer import Consumer, run_consumer_from_stage
from products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stage
from products.batch_exports.backend.temporal.pipeline.producer import Producer
@@ -83,6 +83,7 @@ from products.batch_exports.backend.temporal.queue import RecordBatchQueue, wait
from products.batch_exports.backend.temporal.utils import (
JsonType,
cast_record_batch_schema_json_columns,
+ handle_column_type_mapping_errors,
handle_non_retryable_errors,
)
@@ -656,7 +657,7 @@ def get_redshift_fields_from_record_schema(
pg_type = "SUPER"
else:
- raise TypeError(f"Unsupported type in field '{name}': '{pa_field.type}'")
+ raise ColumnTypeMappingError("Redshift", name, pa_field.type)
pg_schema.append((name, pg_type))
@@ -846,9 +847,10 @@ def _get_table_schemas(
table_schema = [field for field in table_schema if field[0] in record_batch_schema.names]
else:
- table_schema = get_redshift_fields_from_record_schema(
- record_batch_schema, known_super_columns=known_super_columns, use_super=properties_type == "SUPER"
- )
+ with handle_column_type_mapping_errors(model):
+ table_schema = get_redshift_fields_from_record_schema(
+ record_batch_schema, known_super_columns=known_super_columns, use_super=properties_type == "SUPER"
+ )
if properties_type == "SUPER":
# Update stage schema to first load as VARBYTE, and later JSON_PARSE to SUPER
diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
index a28d8972ff7e..17118d594052 100644
--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
@@ -47,6 +47,7 @@ from products.batch_exports.backend.temporal.batch_exports import (
start_batch_export_run,
)
from products.batch_exports.backend.temporal.destinations.utils import get_query_timeout
+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingError
from products.batch_exports.backend.temporal.pipeline.consumer import Consumer, run_consumer_from_stage
from products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stage
from products.batch_exports.backend.temporal.pipeline.producer import Producer
@@ -61,6 +62,7 @@ from products.batch_exports.backend.temporal.queue import RecordBatchQueue, wait
from products.batch_exports.backend.temporal.temporary_file import BatchExportTemporaryFile
from products.batch_exports.backend.temporal.utils import (
JsonType,
+ handle_column_type_mapping_errors,
handle_non_retryable_errors,
make_retryable_with_exponential_backoff,
)
@@ -467,7 +469,10 @@ class SnowflakeField(Field):
@classmethod
def from_arrow_field(cls, field: pa.Field) -> typing.Self:
- snowflake_type = data_type_to_snowflake_type(field.type)
+ try:
+ snowflake_type = data_type_to_snowflake_type(field.type)
+ except TypeError as err:
+ raise ColumnTypeMappingError("Snowflake", field.name, field.type) from err
return cls(field.name, snowflake_type, field.type, field.nullable)
@classmethod
@@ -1537,15 +1542,16 @@ async def insert_into_snowflake_activity_from_stage(
stage_table_name = f"stage_{inputs.table_name}_{data_interval_end_str}_{inputs.team_id}_{attempt}"
requires_merge = model.incremental_mode == IncrementalMode.MERGE
- target_table = SnowflakeTable.from_arrow_schema(
- record_batch_schema,
- name=inputs.table_name,
- table_schema=inputs.schema,
- database=inputs.database,
- primary_key=model.primary_key if requires_merge else (),
- version_key=model.version_key if requires_merge else (),
- stage_prefix=f"{inputs.batch_export_id}/{data_interval_end_str}",
- )
+ with handle_column_type_mapping_errors(model):
+ target_table = SnowflakeTable.from_arrow_schema(
+ record_batch_schema,
+ name=inputs.table_name,
+ table_schema=inputs.schema,
+ database=inputs.database,
+ primary_key=model.primary_key if requires_merge else (),
+ version_key=model.version_key if requires_merge else (),
+ stage_prefix=f"{inputs.batch_export_id}/{data_interval_end_str}",
+ )
if "elements" in target_table:
# `elements` is exported into a 'VARIANT' column, despite it being a
# pa.string.
diff --git a/products/batch_exports/backend/temporal/errors.py b/products/batch_exports/backend/temporal/errors.py
index f37d0901dd35..424dabc9f7bb 100644
--- a/products/batch_exports/backend/temporal/errors.py
+++ b/products/batch_exports/backend/temporal/errors.py
@@ -7,3 +7,30 @@ class MissingRequiredInputsError(RuntimeError):
"""
pass
+
+
+class ColumnTypeMappingError(TypeError):
+ def __init__(self, destination: str, column: str, column_type: object) -> None:
+ self.destination = destination
+ self.column = column
+ self.column_type = str(column_type)
+ super().__init__(f"No {destination} type mapping for column '{column}' with Arrow type '{column_type}'.")
+
+
+class UnsupportedColumnTypeError(Exception):
+ """We tried to export a type that `destination` does not support.
+
+ This should only be possible with HogQL batch exports that take an arbitrary
+ query with arbitrary types. Pre-defined models use well-known queries that
+ should always be compatible.
+ """
+
+ def __init__(self, destination: str, column: str, column_type: object) -> None:
+ self.destination = destination
+ self.column = column
+ self.column_type = str(column_type)
+ super().__init__(
+ f"Column '{column}' has type '{column_type}', which cannot be exported to {destination} "
+ f"without loss of data. Change the query to return a type {destination} supports "
+ "by casting the column to a different type."
+ )
diff --git a/products/batch_exports/backend/temporal/utils.py b/products/batch_exports/backend/temporal/utils.py
index 340fb853f103..77173f252baa 100644
--- a/products/batch_exports/backend/temporal/utils.py
+++ b/products/batch_exports/backend/temporal/utils.py
@@ -14,8 +14,12 @@ from posthog.temporal.common.logger import get_write_only_logger
from products.batch_exports.backend.models.batch_export import BatchExportRun
from products.batch_exports.backend.service import aupdate_batch_export_run
+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingError, UnsupportedColumnTypeError
from products.batch_exports.backend.temporal.pipeline.types import BatchExportResult
+if typing.TYPE_CHECKING:
+ from products.batch_exports.backend.service import AnyBatchExportModel
+
T = typing.TypeVar("T")
LOGGER = get_write_only_logger(__name__)
@@ -321,6 +325,16 @@ def make_retryable_with_exponential_backoff(
return inner
+def handle_column_type_mapping_errors(model: AnyBatchExportModel) -> collections.abc.Iterator[None]:
+ try:
+ yield
+ except ColumnTypeMappingError as error:
+ if model.name == "hogql":
+ raise UnsupportedColumnTypeError(error.destination, error.column, error.column_type) from error
+ raise
+
+
def handle_non_retryable_errors(non_retryable_error_types: typing.Sequence[str]):
"""Decorator to handle non-retryable errors in batch export activities.
@@ -352,11 +366,22 @@ def handle_non_retryable_errors(non_retryable_error_types: typing.Sequence[str])
try:
return await func(*args, **kwargs)
except Exception as e:
+ error_type = type(e).__name__
+ retryable = error_type not in non_retryable_error_types
+ if isinstance(e, (ColumnTypeMappingError, UnsupportedColumnTypeError)):
+ LOGGER.warning(
+ "Unsupported column type in batch export",
+ destination=e.destination,
+ column=e.column,
+ column_type=e.column_type,
+ error_type=error_type,
+ retryable=retryable,
+ )
# If we catch an error at any point during this activity, we check if it's a non-retryable error.
# If it is, we return a BatchExportResult with the error.
# If it's not, we re-raise the error.
# TODO: Use actual exception classes instead of strings.
- if e.__class__.__name__ in non_retryable_error_types:
+ if not retryable:
LOGGER.exception("Non-retryable error caught in activity")
return BatchExportResult.from_exception(e)
LOGGER.exception("Retryable error caught in activity")
diff --git a/products/batch_exports/backend/tests/test_destination_column_types.py b/products/batch_exports/backend/tests/test_destination_column_types.py
new file mode 100644
index 000000000000..9fda56805a42
--- /dev/null
+++ b/products/batch_exports/backend/tests/test_destination_column_types.py
@@ -0,0 +1,155 @@
+import collections.abc
+
+import pytest
+from unittest.mock import patch
+
+import pyarrow as pa
+from structlog.testing import capture_logs
+
+from posthog.temporal.common.logger import get_write_only_logger
+
+from products.batch_exports.backend.service import AnyBatchExportModel, BatchExportHogQLModel, BatchExportPersonsModel
+from products.batch_exports.backend.temporal.destinations import (
+ bigquery_batch_export,
+ databricks_batch_export,
+ postgres_batch_export,
+ redshift_batch_export,
+ snowflake_batch_export,
+)
+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingError, UnsupportedColumnTypeError
+from products.batch_exports.backend.temporal.pipeline.types import BatchExportResult
+from products.batch_exports.backend.temporal.utils import handle_column_type_mapping_errors, handle_non_retryable_errors
+
+
+def _map_bigquery(model: AnyBatchExportModel, schema: pa.Schema) -> object:
+ with handle_column_type_mapping_errors(model):
+ return bigquery_batch_export.BigQueryTable.from_arrow_schema(
+ schema, table_id="t", dataset_id="s", project_id="p", primary_key=(), version_key=()
+ )
+
+
+def _map_snowflake(model: AnyBatchExportModel, schema: pa.Schema) -> object:
+ with handle_column_type_mapping_errors(model):
+ return snowflake_batch_export.SnowflakeTable.from_arrow_schema(
+ schema, name="t", table_schema="s", database="d", primary_key=(), version_key=()
+ )
+
+
+def _map_redshift(model: AnyBatchExportModel, schema: pa.Schema) -> object:
+ return redshift_batch_export._get_table_schemas(model, schema, properties_data_type="varchar")
+
+
+def _map_postgres(model: AnyBatchExportModel, schema: pa.Schema) -> object:
+ return postgres_batch_export._get_table_fields(model, schema)
+
+
+def _map_databricks(model: AnyBatchExportModel, schema: pa.Schema) -> object:
+ return databricks_batch_export._get_databricks_table_settings(model, schema, use_variant_type=False)
+
+
+SCHEMA_MAPPERS: list[
+ tuple[str, collections.abc.Callable[[AnyBatchExportModel, pa.Schema], object], collections.abc.Sequence[str]]
+] = [
+ ("BigQuery", _map_bigquery, bigquery_batch_export.NON_RETRYABLE_ERROR_TYPES),
+ ("Snowflake", _map_snowflake, snowflake_batch_export.NON_RETRYABLE_ERROR_TYPES),
+ ("Redshift", _map_redshift, redshift_batch_export.NON_RETRYABLE_ERROR_TYPES),
+ ("PostgreSQL", _map_postgres, postgres_batch_export.NON_RETRYABLE_ERROR_TYPES),
+ ("Databricks", _map_databricks, databricks_batch_export.NON_RETRYABLE_ERROR_TYPES),
+]
+
+
+ "destination, mapper, non_retryable_error_types", SCHEMA_MAPPERS, ids=[m[0] for m in SCHEMA_MAPPERS]
+)
+ "model, expected_error",
+ [
+ (BatchExportHogQLModel(query="SELECT 1", user_id=1, modifiers={}), UnsupportedColumnTypeError),
+ (BatchExportPersonsModel(), ColumnTypeMappingError),
+ ],
+ ids=["hogql", "persons"],
+)
+ "column_type",
+ [pa.decimal256(76, 9), pa.map_(pa.string(), pa.string()), pa.struct([("a", pa.int64())]), pa.null()],
+ ids=str,
+)
+async def test_schema_mapping_logs_and_retries_an_unsupported_column_type(
+ destination: str,
+ mapper: collections.abc.Callable[[AnyBatchExportModel, pa.Schema], object],
+ non_retryable_error_types: collections.abc.Sequence[str],
+ model: AnyBatchExportModel,
+ expected_error: type[Exception],
+ column_type: pa.DataType,
+) -> None:
+ schema = pa.schema([pa.field("amount", column_type, nullable=True)])
+
+ @handle_non_retryable_errors(non_retryable_error_types)
+ async def map_schema() -> BatchExportResult:
+ mapper(model, schema)
+ return BatchExportResult()
+
+ with (
+ patch("products.batch_exports.backend.temporal.utils.LOGGER", get_write_only_logger(__name__)),
+ capture_logs() as logs,
+ pytest.raises(expected_error) as exc_info,
+ ):
+ await map_schema()
+
+ assert "amount" in str(exc_info.value)
+ assert destination in str(exc_info.value)
+ assert str(column_type) in str(exc_info.value)
+ if expected_error is UnsupportedColumnTypeError:
+ assert isinstance(exc_info.value.__cause__, ColumnTypeMappingError)
+ warnings = [log for log in logs if log["log_level"] == "warning"]
+ assert warnings == [
+ {
+ "event": "Unsupported column type in batch export",
+ "log_level": "warning",
+ "destination": destination,
+ "column": "amount",
+ "column_type": str(column_type),
+ "error_type": expected_error.__name__,
+ "retryable": True,
+ }
+ ]
+
+
+async def test_unsupported_column_warning_reflects_non_retryable_classification() -> None:
+ error = UnsupportedColumnTypeError("PostgreSQL", "amount", pa.null())
+
+ @handle_non_retryable_errors(("UnsupportedColumnTypeError",))
+ async def export() -> BatchExportResult:
+ raise error
+
+ with (
+ patch("products.batch_exports.backend.temporal.utils.LOGGER", get_write_only_logger(__name__)),
+ capture_logs() as logs,
+ ):
+ result = await export()
+
+ assert result == BatchExportResult.from_exception(error)
+ assert [log for log in logs if log["log_level"] == "warning"] == [
+ {
+ "event": "Unsupported column type in batch export",
+ "log_level": "warning",
+ "destination": "PostgreSQL",
+ "column": "amount",
+ "column_type": "null",
+ "error_type": "UnsupportedColumnTypeError",
+ "retryable": False,
+ }
+ ]
+
+
+def test_column_type_mapping_handler_preserves_unrelated_type_errors() -> None:
+ model = BatchExportHogQLModel(query="SELECT 1", user_id=1, modifiers={})
+ error = TypeError("Invalid mapper arguments")
+
+ with pytest.raises(TypeError) as exc_info:
+ with handle_column_type_mapping_errors(model):
+ raise error
+
+ assert exc_info.value is error