Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(all-destinations): Make unsupported column type error non-retryable (#128054)

Tomás Farías Santanadc72db490aa4
diff --git a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
index ca25e186fb16..261f808085b3 100644
--- a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py
@@ -88,6 +88,7 @@ from products.batch_exports.backend.temporal.utils import (
)
NON_RETRYABLE_ERROR_TYPES = (
+ "UnsupportedColumnTypeError",
# Raised on missing permissions.
"Forbidden",
# The stored key file names a token endpoint that is not Google's; only a re-upload fixes it.
diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
index 3b8767a9d465..491e308780e8 100644
--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
@@ -69,6 +69,7 @@ EXTERNAL_LOGGER = get_logger("EXTERNAL")
NON_RETRYABLE_ERROR_TYPES: list[str] = [
+ "UnsupportedColumnTypeError",
# Our own exception when we can't connect to Databricks, usually due to invalid parameters.
"DatabricksConnectionError",
# Raised when we don't have sufficient permissions to perform an operation.
diff --git a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
index 4b827709fb04..7b5cb6aaee4e 100644
--- a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
@@ -85,6 +85,7 @@ LOGGER = get_write_only_logger(__name__)
EXTERNAL_LOGGER = get_logger("EXTERNAL")
NON_RETRYABLE_ERROR_TYPES = (
+ "UnsupportedColumnTypeError",
# Raised on errors that are related to database operation.
# For example: unexpected disconnect, database or other object not found.
"OperationalError",
diff --git a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
index 6209049b19d3..7cbfc280c38d 100644
--- a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py
@@ -91,6 +91,7 @@ LOGGER = get_write_only_logger(__name__)
EXTERNAL_LOGGER = get_logger()
NON_RETRYABLE_ERROR_TYPES = (
+ "UnsupportedColumnTypeError",
# The integration backing this export is missing, of the wrong kind, or
# misconfigured. Retrying can never recover it.
"IntegrationError",
diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
index 9773e676622b..a59923331f4d 100644
--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py
@@ -83,6 +83,7 @@ class NamedBytesIO(io.BytesIO):
CONNECTION_SEMAPHORE = asyncio.Semaphore(value=1)
NON_RETRYABLE_ERROR_TYPES = (
+ "UnsupportedColumnTypeError",
# Raised when we cannot connect to Snowflake.
"DatabaseError",
# Raised by Snowflake when a query cannot be compiled.
diff --git a/products/batch_exports/backend/tests/test_destination_column_types.py b/products/batch_exports/backend/tests/test_destination_column_types.py
index 9fda56805a42..1d2ea6b3d261 100644
--- a/products/batch_exports/backend/tests/test_destination_column_types.py
+++ b/products/batch_exports/backend/tests/test_destination_column_types.py
@@ -75,7 +75,7 @@ SCHEMA_MAPPERS: list[
[pa.decimal256(76, 9), pa.map_(pa.string(), pa.string()), pa.struct([("a", pa.int64())]), pa.null()],
ids=str,
)
-async def test_schema_mapping_logs_and_retries_an_unsupported_column_type(
+async def test_schema_mapping_logs_and_classifies_an_unsupported_column_type(
destination: str,
mapper: collections.abc.Callable[[AnyBatchExportModel, pa.Schema], object],
non_retryable_error_types: collections.abc.Sequence[str],
@@ -84,6 +84,8 @@ async def test_schema_mapping_logs_and_retries_an_unsupported_column_type(
column_type: pa.DataType,
) -> None:
schema = pa.schema([pa.field("amount", column_type, nullable=True)])
+ expected_exception = expected_error(destination, "amount", column_type)
+ retryable = expected_error is ColumnTypeMappingError
@handle_non_retryable_errors(non_retryable_error_types)
async def map_schema() -> BatchExportResult:
@@ -93,15 +95,15 @@ async def test_schema_mapping_logs_and_retries_an_unsupported_column_type(
with (
patch("products.batch_exports.backend.temporal.utils.LOGGER", get_write_only_logger(__name__)),
capture_logs() as logs,
- pytest.raises(expected_error) as exc_info,
):
- await map_schema()
+ if retryable:
+ with pytest.raises(expected_error) as exc_info:
+ await map_schema()
+ assert str(exc_info.value) == str(expected_exception)
+ else:
+ result = await map_schema()
+ assert result == BatchExportResult.from_exception(expected_exception)
- assert "amount" in str(exc_info.value)
- assert destination in str(exc_info.value)
- assert str(column_type) in str(exc_info.value)
- if expected_error is UnsupportedColumnTypeError:
- assert isinstance(exc_info.value.__cause__, ColumnTypeMappingError)
warnings = [log for log in logs if log["log_level"] == "warning"]
assert warnings == [
{
@@ -111,7 +113,7 @@ async def test_schema_mapping_logs_and_retries_an_unsupported_column_type(
"column": "amount",
"column_type": str(column_type),
"error_type": expected_error.__name__,
- "retryable": True,
+ "retryable": retryable,
}
]