:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(batch-exports): log unsupported column types while retrying (#121670)
Phill5c9c06c76b7f
diff --git a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.pyindex 07fbb41588e2..50ecaf739ad9 100644--- a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py@@ -63,6 +63,7 @@ from products.batch_exports.backend.temporal.batch_exports import (is_over_billing_limit_error,start_batch_export_run,)+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingErrorfrom products.batch_exports.backend.temporal.pipeline.consumer import Consumerfrom products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stagefrom products.batch_exports.backend.temporal.pipeline.producer import Producer@@ -81,6 +82,7 @@ from products.batch_exports.backend.temporal.queue import ()from products.batch_exports.backend.temporal.utils import (JsonType,+ handle_column_type_mapping_errors,handle_non_retryable_errors,make_retryable_with_exponential_backoff,)@@ -272,7 +274,10 @@ class BigQueryField(Field):@classmethoddef from_arrow_field(cls, field: pa.Field) -> typing.Self:- type = data_type_to_bigquery_type(field.type)+ try:+ type = data_type_to_bigquery_type(field.type)+ except ValueError as err:+ raise ColumnTypeMappingError("BigQuery", field.name, field.type) from errreturn cls(field.name, type, nullable=field.nullable)@classmethod@@ -1577,14 +1582,15 @@ async def insert_into_bigquery_activity_from_stage(inputs: BigQueryInsertInputs)json_fields = set()requires_merge = model.incremental_mode == IncrementalMode.MERGE- target_table = BigQueryTable.from_arrow_schema(- record_batch_schema,- table_id=inputs.table_id,- project_id=project_id,- dataset_id=inputs.dataset_id,- primary_key=model.primary_key if requires_merge else (),- version_key=model.version_key if requires_merge else (),- )+ with handle_column_type_mapping_errors(model):+ target_table = BigQueryTable.from_arrow_schema(+ record_batch_schema,+ table_id=inputs.table_id,+ project_id=project_id,+ dataset_id=inputs.dataset_id,+ primary_key=model.primary_key if requires_merge else (),+ version_key=model.version_key if requires_merge else (),+ )data_interval_end_str = dt.datetime.fromisoformat(inputs.data_interval_end).strftime("%Y-%m-%d_%H-%M-%S")attempt = activity.info().attemptdiff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.pyindex 557bf5323e99..d5291ab64b17 100644--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py@@ -47,6 +47,7 @@ from products.batch_exports.backend.temporal.batch_exports import (is_over_billing_limit_error,start_batch_export_run,)+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingErrorfrom products.batch_exports.backend.temporal.pipeline.consumer import Consumer, run_consumer_from_stagefrom products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stagefrom products.batch_exports.backend.temporal.pipeline.producer import Producer@@ -58,6 +59,7 @@ from products.batch_exports.backend.temporal.pipeline.types import BatchExportRefrom products.batch_exports.backend.temporal.queue import RecordBatchQueue, wait_for_schema_or_producerfrom products.batch_exports.backend.temporal.utils import (JsonType,+ handle_column_type_mapping_errors,handle_non_retryable_errors,make_retryable_with_exponential_backoff,)@@ -1037,7 +1039,7 @@ def _get_databricks_fields_from_record_schema(is_variant = pa_field.name in known_variant_columnsdatabricks_type = _get_databricks_field_type(pa_field.type, is_variant)if databricks_type is None:- raise TypeError(f"Unsupported type in field '{name}': '{databricks_type}'")+ raise ColumnTypeMappingError("Databricks", name, pa_field.type)databricks_schema.append((name, databricks_type))return databricks_schema@@ -1110,10 +1112,11 @@ def _get_databricks_table_settings(if staged_fields:table_fields = staged_fieldselse:- table_fields = _get_databricks_fields_from_record_schema(- record_batch_schema,- known_variant_columns=known_variant_columns,- )+ with handle_column_type_mapping_errors(model):+ table_fields = _get_databricks_fields_from_record_schema(+ record_batch_schema,+ known_variant_columns=known_variant_columns,+ )return TableSettings(table_fields, record_batch_schema, known_variant_columns)diff --git a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.pyindex 67d9e52be2f3..ed059fc75ab3 100644--- a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py@@ -53,6 +53,7 @@ from products.batch_exports.backend.temporal.batch_exports import (start_batch_export_run,)from products.batch_exports.backend.temporal.destinations.utils import get_query_timeout+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingErrorfrom products.batch_exports.backend.temporal.pipeline.consumer import Consumer, run_consumer_from_stagefrom products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stagefrom products.batch_exports.backend.temporal.pipeline.producer import Producer@@ -61,6 +62,7 @@ from products.batch_exports.backend.temporal.pipeline.types import BatchExportRefrom products.batch_exports.backend.temporal.queue import RecordBatchQueue, wait_for_schema_or_producerfrom products.batch_exports.backend.temporal.utils import (JsonType,+ handle_column_type_mapping_errors,handle_non_retryable_errors,make_retryable_with_exponential_backoff,)@@ -838,7 +840,7 @@ def get_postgres_fields_from_record_schema(pg_type = "TEXT[]"else:- raise TypeError(f"Unsupported type in field '{name}': '{pa_field.type}'")+ raise ColumnTypeMappingError("PostgreSQL", name, pa_field.type)pg_schema.append((name, pg_type))@@ -934,10 +936,11 @@ def _get_table_fields(# A retry can consume files staged before a new default column was added.return [field for field in table_fields if field[0] in record_batch_schema.names]else:- return get_postgres_fields_from_record_schema(- record_batch_schema,- known_json_columns=["properties", "set", "set_once", "person_properties"],- )+ with handle_column_type_mapping_errors(model):+ return get_postgres_fields_from_record_schema(+ record_batch_schema,+ known_json_columns=["properties", "set", "set_once", "person_properties"],+ )class PostgreSQLIntegrationNotFoundError(Exception):diff --git a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.pyindex a46287eda88d..a1aceee13e81 100644--- a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py@@ -70,7 +70,7 @@ from products.batch_exports.backend.temporal.destinations.s3_batch_export imports3_client,)from products.batch_exports.backend.temporal.destinations.utils import get_absolute_key_prefix-from products.batch_exports.backend.temporal.errors import MissingRequiredInputsError+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingError, MissingRequiredInputsErrorfrom products.batch_exports.backend.temporal.pipeline.consumer import Consumer, run_consumer_from_stagefrom products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stagefrom products.batch_exports.backend.temporal.pipeline.producer import Producer@@ -83,6 +83,7 @@ from products.batch_exports.backend.temporal.queue import RecordBatchQueue, waitfrom products.batch_exports.backend.temporal.utils import (JsonType,cast_record_batch_schema_json_columns,+ handle_column_type_mapping_errors,handle_non_retryable_errors,)@@ -656,7 +657,7 @@ def get_redshift_fields_from_record_schema(pg_type = "SUPER"else:- raise TypeError(f"Unsupported type in field '{name}': '{pa_field.type}'")+ raise ColumnTypeMappingError("Redshift", name, pa_field.type)pg_schema.append((name, pg_type))@@ -846,9 +847,10 @@ def _get_table_schemas(table_schema = [field for field in table_schema if field[0] in record_batch_schema.names]else:- table_schema = get_redshift_fields_from_record_schema(- record_batch_schema, known_super_columns=known_super_columns, use_super=properties_type == "SUPER"- )+ with handle_column_type_mapping_errors(model):+ table_schema = get_redshift_fields_from_record_schema(+ record_batch_schema, known_super_columns=known_super_columns, use_super=properties_type == "SUPER"+ )if properties_type == "SUPER":# Update stage schema to first load as VARBYTE, and later JSON_PARSE to SUPERdiff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.pyindex a28d8972ff7e..17118d594052 100644--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py@@ -47,6 +47,7 @@ from products.batch_exports.backend.temporal.batch_exports import (start_batch_export_run,)from products.batch_exports.backend.temporal.destinations.utils import get_query_timeout+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingErrorfrom products.batch_exports.backend.temporal.pipeline.consumer import Consumer, run_consumer_from_stagefrom products.batch_exports.backend.temporal.pipeline.entrypoint import execute_batch_export_using_internal_stagefrom products.batch_exports.backend.temporal.pipeline.producer import Producer@@ -61,6 +62,7 @@ from products.batch_exports.backend.temporal.queue import RecordBatchQueue, waitfrom products.batch_exports.backend.temporal.temporary_file import BatchExportTemporaryFilefrom products.batch_exports.backend.temporal.utils import (JsonType,+ handle_column_type_mapping_errors,handle_non_retryable_errors,make_retryable_with_exponential_backoff,)@@ -467,7 +469,10 @@ class SnowflakeField(Field):@classmethoddef from_arrow_field(cls, field: pa.Field) -> typing.Self:- snowflake_type = data_type_to_snowflake_type(field.type)+ try:+ snowflake_type = data_type_to_snowflake_type(field.type)+ except TypeError as err:+ raise ColumnTypeMappingError("Snowflake", field.name, field.type) from errreturn cls(field.name, snowflake_type, field.type, field.nullable)@classmethod@@ -1537,15 +1542,16 @@ async def insert_into_snowflake_activity_from_stage(stage_table_name = f"stage_{inputs.table_name}_{data_interval_end_str}_{inputs.team_id}_{attempt}"requires_merge = model.incremental_mode == IncrementalMode.MERGE- target_table = SnowflakeTable.from_arrow_schema(- record_batch_schema,- name=inputs.table_name,- table_schema=inputs.schema,- database=inputs.database,- primary_key=model.primary_key if requires_merge else (),- version_key=model.version_key if requires_merge else (),- stage_prefix=f"{inputs.batch_export_id}/{data_interval_end_str}",- )+ with handle_column_type_mapping_errors(model):+ target_table = SnowflakeTable.from_arrow_schema(+ record_batch_schema,+ name=inputs.table_name,+ table_schema=inputs.schema,+ database=inputs.database,+ primary_key=model.primary_key if requires_merge else (),+ version_key=model.version_key if requires_merge else (),+ stage_prefix=f"{inputs.batch_export_id}/{data_interval_end_str}",+ )if "elements" in target_table:# `elements` is exported into a 'VARIANT' column, despite it being a# pa.string.diff --git a/products/batch_exports/backend/temporal/errors.py b/products/batch_exports/backend/temporal/errors.pyindex f37d0901dd35..424dabc9f7bb 100644--- a/products/batch_exports/backend/temporal/errors.py+++ b/products/batch_exports/backend/temporal/errors.py@@ -7,3 +7,30 @@ class MissingRequiredInputsError(RuntimeError):"""pass+++class ColumnTypeMappingError(TypeError):+ def __init__(self, destination: str, column: str, column_type: object) -> None:+ self.destination = destination+ self.column = column+ self.column_type = str(column_type)+ super().__init__(f"No {destination} type mapping for column '{column}' with Arrow type '{column_type}'.")+++class UnsupportedColumnTypeError(Exception):+ """We tried to export a type that `destination` does not support.++ This should only be possible with HogQL batch exports that take an arbitrary+ query with arbitrary types. Pre-defined models use well-known queries that+ should always be compatible.+ """++ def __init__(self, destination: str, column: str, column_type: object) -> None:+ self.destination = destination+ self.column = column+ self.column_type = str(column_type)+ super().__init__(+ f"Column '{column}' has type '{column_type}', which cannot be exported to {destination} "+ f"without loss of data. Change the query to return a type {destination} supports "+ "by casting the column to a different type."+ )diff --git a/products/batch_exports/backend/temporal/utils.py b/products/batch_exports/backend/temporal/utils.pyindex 340fb853f103..77173f252baa 100644--- a/products/batch_exports/backend/temporal/utils.py+++ b/products/batch_exports/backend/temporal/utils.py@@ -14,8 +14,12 @@ from posthog.temporal.common.logger import get_write_only_loggerfrom products.batch_exports.backend.models.batch_export import BatchExportRunfrom products.batch_exports.backend.service import aupdate_batch_export_run+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingError, UnsupportedColumnTypeErrorfrom products.batch_exports.backend.temporal.pipeline.types import BatchExportResult+if typing.TYPE_CHECKING:+ from products.batch_exports.backend.service import AnyBatchExportModel+T = typing.TypeVar("T")LOGGER = get_write_only_logger(__name__)@@ -321,6 +325,16 @@ def make_retryable_with_exponential_backoff(return inner+def handle_column_type_mapping_errors(model: AnyBatchExportModel) -> collections.abc.Iterator[None]:+ try:+ yield+ except ColumnTypeMappingError as error:+ if model.name == "hogql":+ raise UnsupportedColumnTypeError(error.destination, error.column, error.column_type) from error+ raise++def handle_non_retryable_errors(non_retryable_error_types: typing.Sequence[str]):"""Decorator to handle non-retryable errors in batch export activities.@@ -352,11 +366,22 @@ def handle_non_retryable_errors(non_retryable_error_types: typing.Sequence[str])try:return await func(*args, **kwargs)except Exception as e:+ error_type = type(e).__name__+ retryable = error_type not in non_retryable_error_types+ if isinstance(e, (ColumnTypeMappingError, UnsupportedColumnTypeError)):+ LOGGER.warning(+ "Unsupported column type in batch export",+ destination=e.destination,+ column=e.column,+ column_type=e.column_type,+ error_type=error_type,+ retryable=retryable,+ )# If we catch an error at any point during this activity, we check if it's a non-retryable error.# If it is, we return a BatchExportResult with the error.# If it's not, we re-raise the error.# TODO: Use actual exception classes instead of strings.- if e.__class__.__name__ in non_retryable_error_types:+ if not retryable:LOGGER.exception("Non-retryable error caught in activity")return BatchExportResult.from_exception(e)LOGGER.exception("Retryable error caught in activity")diff --git a/products/batch_exports/backend/tests/test_destination_column_types.py b/products/batch_exports/backend/tests/test_destination_column_types.pynew file mode 100644index 000000000000..9fda56805a42--- /dev/null+++ b/products/batch_exports/backend/tests/test_destination_column_types.py@@ -0,0 +1,155 @@+import collections.abc++import pytest+from unittest.mock import patch++import pyarrow as pa+from structlog.testing import capture_logs++from posthog.temporal.common.logger import get_write_only_logger++from products.batch_exports.backend.service import AnyBatchExportModel, BatchExportHogQLModel, BatchExportPersonsModel+from products.batch_exports.backend.temporal.destinations import (+ bigquery_batch_export,+ databricks_batch_export,+ postgres_batch_export,+ redshift_batch_export,+ snowflake_batch_export,+)+from products.batch_exports.backend.temporal.errors import ColumnTypeMappingError, UnsupportedColumnTypeError+from products.batch_exports.backend.temporal.pipeline.types import BatchExportResult+from products.batch_exports.backend.temporal.utils import handle_column_type_mapping_errors, handle_non_retryable_errors+++def _map_bigquery(model: AnyBatchExportModel, schema: pa.Schema) -> object:+ with handle_column_type_mapping_errors(model):+ return bigquery_batch_export.BigQueryTable.from_arrow_schema(+ schema, table_id="t", dataset_id="s", project_id="p", primary_key=(), version_key=()+ )+++def _map_snowflake(model: AnyBatchExportModel, schema: pa.Schema) -> object:+ with handle_column_type_mapping_errors(model):+ return snowflake_batch_export.SnowflakeTable.from_arrow_schema(+ schema, name="t", table_schema="s", database="d", primary_key=(), version_key=()+ )+++def _map_redshift(model: AnyBatchExportModel, schema: pa.Schema) -> object:+ return redshift_batch_export._get_table_schemas(model, schema, properties_data_type="varchar")+++def _map_postgres(model: AnyBatchExportModel, schema: pa.Schema) -> object:+ return postgres_batch_export._get_table_fields(model, schema)+++def _map_databricks(model: AnyBatchExportModel, schema: pa.Schema) -> object:+ return databricks_batch_export._get_databricks_table_settings(model, schema, use_variant_type=False)+++SCHEMA_MAPPERS: list[+ tuple[str, collections.abc.Callable[[AnyBatchExportModel, pa.Schema], object], collections.abc.Sequence[str]]+] = [+ ("BigQuery", _map_bigquery, bigquery_batch_export.NON_RETRYABLE_ERROR_TYPES),+ ("Snowflake", _map_snowflake, snowflake_batch_export.NON_RETRYABLE_ERROR_TYPES),+ ("Redshift", _map_redshift, redshift_batch_export.NON_RETRYABLE_ERROR_TYPES),+ ("PostgreSQL", _map_postgres, postgres_batch_export.NON_RETRYABLE_ERROR_TYPES),+ ("Databricks", _map_databricks, databricks_batch_export.NON_RETRYABLE_ERROR_TYPES),+]+++ "destination, mapper, non_retryable_error_types", SCHEMA_MAPPERS, ids=[m[0] for m in SCHEMA_MAPPERS]+)+ "model, expected_error",+ [+ (BatchExportHogQLModel(query="SELECT 1", user_id=1, modifiers={}), UnsupportedColumnTypeError),+ (BatchExportPersonsModel(), ColumnTypeMappingError),+ ],+ ids=["hogql", "persons"],+)+ "column_type",+ [pa.decimal256(76, 9), pa.map_(pa.string(), pa.string()), pa.struct([("a", pa.int64())]), pa.null()],+ ids=str,+)+async def test_schema_mapping_logs_and_retries_an_unsupported_column_type(+ destination: str,+ mapper: collections.abc.Callable[[AnyBatchExportModel, pa.Schema], object],+ non_retryable_error_types: collections.abc.Sequence[str],+ model: AnyBatchExportModel,+ expected_error: type[Exception],+ column_type: pa.DataType,+) -> None:+ schema = pa.schema([pa.field("amount", column_type, nullable=True)])++ @handle_non_retryable_errors(non_retryable_error_types)+ async def map_schema() -> BatchExportResult:+ mapper(model, schema)+ return BatchExportResult()++ with (+ patch("products.batch_exports.backend.temporal.utils.LOGGER", get_write_only_logger(__name__)),+ capture_logs() as logs,+ pytest.raises(expected_error) as exc_info,+ ):+ await map_schema()++ assert "amount" in str(exc_info.value)+ assert destination in str(exc_info.value)+ assert str(column_type) in str(exc_info.value)+ if expected_error is UnsupportedColumnTypeError:+ assert isinstance(exc_info.value.__cause__, ColumnTypeMappingError)+ warnings = [log for log in logs if log["log_level"] == "warning"]+ assert warnings == [+ {+ "event": "Unsupported column type in batch export",+ "log_level": "warning",+ "destination": destination,+ "column": "amount",+ "column_type": str(column_type),+ "error_type": expected_error.__name__,+ "retryable": True,+ }+ ]+++async def test_unsupported_column_warning_reflects_non_retryable_classification() -> None:+ error = UnsupportedColumnTypeError("PostgreSQL", "amount", pa.null())++ @handle_non_retryable_errors(("UnsupportedColumnTypeError",))+ async def export() -> BatchExportResult:+ raise error++ with (+ patch("products.batch_exports.backend.temporal.utils.LOGGER", get_write_only_logger(__name__)),+ capture_logs() as logs,+ ):+ result = await export()++ assert result == BatchExportResult.from_exception(error)+ assert [log for log in logs if log["log_level"] == "warning"] == [+ {+ "event": "Unsupported column type in batch export",+ "log_level": "warning",+ "destination": "PostgreSQL",+ "column": "amount",+ "column_type": "null",+ "error_type": "UnsupportedColumnTypeError",+ "retryable": False,+ }+ ]+++def test_column_type_mapping_handler_preserves_unrelated_type_errors() -> None:+ model = BatchExportHogQLModel(query="SELECT 1", user_id=1, modifiers={})+ error = TypeError("Invalid mapper arguments")++ with pytest.raises(TypeError) as exc_info:+ with handle_column_type_mapping_errors(model):+ raise error++ assert exc_info.value is error