:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): stop none entries hiding friendly sync errors (#129955)
Tom Owers0f530f7fd6dd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.pyindex d939807c0a64..c6251c3f3807 100644--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py@@ -477,6 +477,23 @@ def _is_app_db_failure(internal_error: str) -> bool:return normalized.startswith(APP_DB_ERROR_PREFIX) and READ_ONLY_TRANSACTION_PHRASE in normalized+def _non_retryable_error_message(non_retryable_errors: dict[str, str | None], internal_error: str) -> str | None:+ """Customer-facing copy for a non-retryable failure: the first matching entry that has one.++ A `None` entry only marks the error non-retryable. Temporal prefixes the driver's class name, so+ a class-name key such as `InsufficientPrivilege` matches the same error as a later key that+ carries the message, and it must not hide that message.+ """+ return next(+ (+ message+ for pattern, message in non_retryable_errors.items()+ if message is not None and error_message_matches(internal_error, [pattern])+ ),+ None,+ )++def _transient_error_message(internal_error: str) -> str | None:"""Customer-facing copy for a transient failure that outlived its retries, if we have any.@@ -647,15 +664,10 @@ async def _update_job_status(inputs: UpdateExternalDataJobStatusInputs, logger:)if has_non_retryable_error:- friendly_errors = [- friendly_error- for error, friendly_error in non_retryable_errors.items()- if error_message_matches(internal_error_normalized, [error])- ]-- if friendly_errors and friendly_errors[0] is not None:- logger.exception(friendly_errors[0])- inputs.latest_error = friendly_errors[0]+ friendly_error = _non_retryable_error_message(non_retryable_errors, internal_error_normalized)+ if friendly_error is not None:+ logger.exception(friendly_error)+ inputs.latest_error = friendly_error# Computed after the friendly error so the teardown records the same message# the job will show. Excluding this workflow keeps the disable's teardown fromdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex 25bcdcb0d45c..4a64a84a918b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -1741,9 +1741,7 @@ class TestPostgresSourceNonRetryableErrors:['infinite recursion detected in policy for relation "list_members"',# Temporal-wrapped form matches both this pattern and the "InvalidObjectDefinition"- # class-name key; `update_external_data_job_model` takes the friendly message from the- # *first* matching dict entry, so the specific pattern must be ordered before the- # class-name key or its `None` value silently shadows this actionable message.+ # class-name key, so the first matching entry must carry the actionable message.'InvalidObjectDefinition: infinite recursion detected in policy for relation "orders"',],)diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.pyindex cd76089808d7..94c85a479cf5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py@@ -30,14 +30,20 @@ from products.warehouse_sources.backend.temporal.data_imports.external_data_jobTRANSIENT_SSH_TUNNEL_CONNECT_DROPPED_MESSAGE,TRANSIENT_VENDOR_UNAVAILABLE_MESSAGE,UNEXPECTED_ERROR_MESSAGE,+ Any_Source_Errors,UpdateExternalDataJobStatusInputs,_customer_facing_error,_is_app_db_failure,+ _non_retryable_error_message,trigger_schedule_buffer_one_activity,update_external_data_job_model,)+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.postgres import (+ XMIN_AS_INCREMENTAL_FIELD_ERROR,+)from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (_CONNECTION_LIMIT_EXHAUSTED_MESSAGE,+ PostgresSource,)from products.warehouse_sources.backend.temporal.data_imports.sources.redshift.source import RedshiftSourcefrom products.warehouse_sources.backend.types import ExternalDataJobStatus, ExternalDataSourceType@@ -113,6 +119,56 @@ class TestIsAppDbFailure(SimpleTestCase):assert _is_app_db_failure(str(ApplicationError(message, type=exc_type))) is expected+class TestNonRetryableErrorMessage(SimpleTestCase):+ @parameterized.expand(+ [+ # Each class-name or broad key below has a None value and sits before the entry that+ # carries the message, so a first-match pick showed the customer the raw driver text.+ (+ "postgres_table_permission",+ PostgresSource,+ "InsufficientPrivilege",+ "permission denied for table orders",+ "GRANT SELECT",+ ),+ (+ "postgres_schema_permission",+ PostgresSource,+ "InsufficientPrivilege",+ "permission denied for schema analytics",+ "USAGE",+ ),+ (+ "redshift_relation_permission",+ RedshiftSource,+ "InsufficientPrivilege",+ "permission denied for relation orders",+ "Redshift reported",+ ),+ (+ "postgres_xmin_as_incremental_field",+ PostgresSource,+ "ValueError",+ XMIN_AS_INCREMENTAL_FIELD_ERROR,+ "choose a different incremental field",+ ),+ ]+ )+ def test_a_none_entry_does_not_hide_a_later_message(+ self, _name: str, source_cls: type, exc_type: str, message: str, expected_fragment: str+ ) -> None:+ errors = {**Any_Source_Errors, **source_cls().get_non_retryable_errors()}+ result = _non_retryable_error_message(errors, str(ApplicationError(message, type=exc_type)))+ assert result is not None+ assert expected_fragment in result+ assert "orders" not in result and "analytics" not in result++ def test_an_error_only_none_entries_match_keeps_no_message(self) -> None:+ errors = {**Any_Source_Errors, **PostgresSource().get_non_retryable_errors()}+ internal_error = str(ApplicationError("table orders is gone", type="NoSuchTableError"))+ assert _non_retryable_error_message(errors, internal_error) is None++class TestTriggerScheduleBufferOneActivity(BaseTest):@parameterized.expand([("billable", True), ("non_billable", False)])def test_worker_shutdown_retry_keeps_the_runs_billable_flag(self, _name: str, billable: bool):