Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): stop none entries hiding friendly sync errors (#129955)

Tom Owers0f530f7fd6dd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
index d939807c0a64..c6251c3f3807 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
@@ -477,6 +477,23 @@ def _is_app_db_failure(internal_error: str) -> bool:
return normalized.startswith(APP_DB_ERROR_PREFIX) and READ_ONLY_TRANSACTION_PHRASE in normalized
+def _non_retryable_error_message(non_retryable_errors: dict[str, str | None], internal_error: str) -> str | None:
+ """Customer-facing copy for a non-retryable failure: the first matching entry that has one.
+
+ A `None` entry only marks the error non-retryable. Temporal prefixes the driver's class name, so
+ a class-name key such as `InsufficientPrivilege` matches the same error as a later key that
+ carries the message, and it must not hide that message.
+ """
+ return next(
+ (
+ message
+ for pattern, message in non_retryable_errors.items()
+ if message is not None and error_message_matches(internal_error, [pattern])
+ ),
+ None,
+ )
+
+
def _transient_error_message(internal_error: str) -> str | None:
"""Customer-facing copy for a transient failure that outlived its retries, if we have any.
@@ -647,15 +664,10 @@ async def _update_job_status(inputs: UpdateExternalDataJobStatusInputs, logger:
)
if has_non_retryable_error:
- friendly_errors = [
- friendly_error
- for error, friendly_error in non_retryable_errors.items()
- if error_message_matches(internal_error_normalized, [error])
- ]
-
- if friendly_errors and friendly_errors[0] is not None:
- logger.exception(friendly_errors[0])
- inputs.latest_error = friendly_errors[0]
+ friendly_error = _non_retryable_error_message(non_retryable_errors, internal_error_normalized)
+ if friendly_error is not None:
+ logger.exception(friendly_error)
+ inputs.latest_error = friendly_error
# Computed after the friendly error so the teardown records the same message
# the job will show. Excluding this workflow keeps the disable's teardown from
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index 25bcdcb0d45c..4a64a84a918b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -1741,9 +1741,7 @@ class TestPostgresSourceNonRetryableErrors:
[
'infinite recursion detected in policy for relation "list_members"',
# Temporal-wrapped form matches both this pattern and the "InvalidObjectDefinition"
- # class-name key; `update_external_data_job_model` takes the friendly message from the
- # *first* matching dict entry, so the specific pattern must be ordered before the
- # class-name key or its `None` value silently shadows this actionable message.
+ # class-name key, so the first matching entry must carry the actionable message.
'InvalidObjectDefinition: infinite recursion detected in policy for relation "orders"',
],
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
index cd76089808d7..94c85a479cf5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
@@ -30,14 +30,20 @@ from products.warehouse_sources.backend.temporal.data_imports.external_data_job
TRANSIENT_SSH_TUNNEL_CONNECT_DROPPED_MESSAGE,
TRANSIENT_VENDOR_UNAVAILABLE_MESSAGE,
UNEXPECTED_ERROR_MESSAGE,
+ Any_Source_Errors,
UpdateExternalDataJobStatusInputs,
_customer_facing_error,
_is_app_db_failure,
+ _non_retryable_error_message,
trigger_schedule_buffer_one_activity,
update_external_data_job_model,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.postgres import (
+ XMIN_AS_INCREMENTAL_FIELD_ERROR,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (
_CONNECTION_LIMIT_EXHAUSTED_MESSAGE,
+ PostgresSource,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.redshift.source import RedshiftSource
from products.warehouse_sources.backend.types import ExternalDataJobStatus, ExternalDataSourceType
@@ -113,6 +119,56 @@ class TestIsAppDbFailure(SimpleTestCase):
assert _is_app_db_failure(str(ApplicationError(message, type=exc_type))) is expected
+class TestNonRetryableErrorMessage(SimpleTestCase):
+ @parameterized.expand(
+ [
+ # Each class-name or broad key below has a None value and sits before the entry that
+ # carries the message, so a first-match pick showed the customer the raw driver text.
+ (
+ "postgres_table_permission",
+ PostgresSource,
+ "InsufficientPrivilege",
+ "permission denied for table orders",
+ "GRANT SELECT",
+ ),
+ (
+ "postgres_schema_permission",
+ PostgresSource,
+ "InsufficientPrivilege",
+ "permission denied for schema analytics",
+ "USAGE",
+ ),
+ (
+ "redshift_relation_permission",
+ RedshiftSource,
+ "InsufficientPrivilege",
+ "permission denied for relation orders",
+ "Redshift reported",
+ ),
+ (
+ "postgres_xmin_as_incremental_field",
+ PostgresSource,
+ "ValueError",
+ XMIN_AS_INCREMENTAL_FIELD_ERROR,
+ "choose a different incremental field",
+ ),
+ ]
+ )
+ def test_a_none_entry_does_not_hide_a_later_message(
+ self, _name: str, source_cls: type, exc_type: str, message: str, expected_fragment: str
+ ) -> None:
+ errors = {**Any_Source_Errors, **source_cls().get_non_retryable_errors()}
+ result = _non_retryable_error_message(errors, str(ApplicationError(message, type=exc_type)))
+ assert result is not None
+ assert expected_fragment in result
+ assert "orders" not in result and "analytics" not in result
+
+ def test_an_error_only_none_entries_match_keeps_no_message(self) -> None:
+ errors = {**Any_Source_Errors, **PostgresSource().get_non_retryable_errors()}
+ internal_error = str(ApplicationError("table orders is gone", type="NoSuchTableError"))
+ assert _non_retryable_error_message(errors, internal_error) is None
+
+
class TestTriggerScheduleBufferOneActivity(BaseTest):
@parameterized.expand([("billable", True), ("non_billable", False)])
def test_worker_shutdown_retry_keeps_the_runs_billable_flag(self, _name: str, billable: bool):