:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(all-destinations): Make unsupported column type error non-retryable (#128054)
Tomás Farías Santanadc72db490aa4
diff --git a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.pyindex ca25e186fb16..261f808085b3 100644--- a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py@@ -88,6 +88,7 @@ from products.batch_exports.backend.temporal.utils import ()NON_RETRYABLE_ERROR_TYPES = (+ "UnsupportedColumnTypeError",# Raised on missing permissions."Forbidden",# The stored key file names a token endpoint that is not Google's; only a re-upload fixes it.diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.pyindex 3b8767a9d465..491e308780e8 100644--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py@@ -69,6 +69,7 @@ EXTERNAL_LOGGER = get_logger("EXTERNAL")NON_RETRYABLE_ERROR_TYPES: list[str] = [+ "UnsupportedColumnTypeError",# Our own exception when we can't connect to Databricks, usually due to invalid parameters."DatabricksConnectionError",# Raised when we don't have sufficient permissions to perform an operation.diff --git a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.pyindex 4b827709fb04..7b5cb6aaee4e 100644--- a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py@@ -85,6 +85,7 @@ LOGGER = get_write_only_logger(__name__)EXTERNAL_LOGGER = get_logger("EXTERNAL")NON_RETRYABLE_ERROR_TYPES = (+ "UnsupportedColumnTypeError",# Raised on errors that are related to database operation.# For example: unexpected disconnect, database or other object not found."OperationalError",diff --git a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.pyindex 6209049b19d3..7cbfc280c38d 100644--- a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py@@ -91,6 +91,7 @@ LOGGER = get_write_only_logger(__name__)EXTERNAL_LOGGER = get_logger()NON_RETRYABLE_ERROR_TYPES = (+ "UnsupportedColumnTypeError",# The integration backing this export is missing, of the wrong kind, or# misconfigured. Retrying can never recover it."IntegrationError",diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.pyindex 9773e676622b..a59923331f4d 100644--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py@@ -83,6 +83,7 @@ class NamedBytesIO(io.BytesIO):CONNECTION_SEMAPHORE = asyncio.Semaphore(value=1)NON_RETRYABLE_ERROR_TYPES = (+ "UnsupportedColumnTypeError",# Raised when we cannot connect to Snowflake."DatabaseError",# Raised by Snowflake when a query cannot be compiled.diff --git a/products/batch_exports/backend/tests/test_destination_column_types.py b/products/batch_exports/backend/tests/test_destination_column_types.pyindex 9fda56805a42..1d2ea6b3d261 100644--- a/products/batch_exports/backend/tests/test_destination_column_types.py+++ b/products/batch_exports/backend/tests/test_destination_column_types.py@@ -75,7 +75,7 @@ SCHEMA_MAPPERS: list[[pa.decimal256(76, 9), pa.map_(pa.string(), pa.string()), pa.struct([("a", pa.int64())]), pa.null()],ids=str,)-async def test_schema_mapping_logs_and_retries_an_unsupported_column_type(+async def test_schema_mapping_logs_and_classifies_an_unsupported_column_type(destination: str,mapper: collections.abc.Callable[[AnyBatchExportModel, pa.Schema], object],non_retryable_error_types: collections.abc.Sequence[str],@@ -84,6 +84,8 @@ async def test_schema_mapping_logs_and_retries_an_unsupported_column_type(column_type: pa.DataType,) -> None:schema = pa.schema([pa.field("amount", column_type, nullable=True)])+ expected_exception = expected_error(destination, "amount", column_type)+ retryable = expected_error is ColumnTypeMappingError@handle_non_retryable_errors(non_retryable_error_types)async def map_schema() -> BatchExportResult:@@ -93,15 +95,15 @@ async def test_schema_mapping_logs_and_retries_an_unsupported_column_type(with (patch("products.batch_exports.backend.temporal.utils.LOGGER", get_write_only_logger(__name__)),capture_logs() as logs,- pytest.raises(expected_error) as exc_info,):- await map_schema()+ if retryable:+ with pytest.raises(expected_error) as exc_info:+ await map_schema()+ assert str(exc_info.value) == str(expected_exception)+ else:+ result = await map_schema()+ assert result == BatchExportResult.from_exception(expected_exception)- assert "amount" in str(exc_info.value)- assert destination in str(exc_info.value)- assert str(column_type) in str(exc_info.value)- if expected_error is UnsupportedColumnTypeError:- assert isinstance(exc_info.value.__cause__, ColumnTypeMappingError)warnings = [log for log in logs if log["log_level"] == "warning"]assert warnings == [{@@ -111,7 +113,7 @@ async def test_schema_mapping_logs_and_retries_an_unsupported_column_type("column": "amount","column_type": str(column_type),"error_type": expected_error.__name__,- "retryable": True,+ "retryable": retryable,}]