:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(trino): stop reporting worker-node blips as exceptions (#118351)
Tom Owers8294182f4f28
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.pyindex 26d96bec1c0a..9ad9bc5ef897 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py@@ -28,6 +28,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.trino.trinTRINO_KNOWN_ERROR_MESSAGES,TRINO_SCHEMA_NOT_FOUND_ERROR,TRINO_TABLE_NOT_FOUND_ERROR,+ TRINO_WORKER_NODE_ERROR,TrinoImplementation,TrinoSchemaDiscoveryError,connect_trino,@@ -283,3 +284,8 @@ class TrinoSource(SQLSource[TrinoSourceConfig], ValidateDatabaseHostMixin):"name=COLUMN_NOT_FOUND": TRINO_COLUMN_NOT_FOUND_ERROR,TrinoSchemaDiscoveryError.__name__: None,}++ def get_retryable_errors(self) -> set[str]:+ # The coordinator only raises this after exhausting its own attempts to reach the worker,+ # so it reaches us already a self-recovering vendor-side blip.+ return {TRINO_WORKER_NODE_ERROR}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.pyindex 46335d1aa7f4..52d7bf335858 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py@@ -685,6 +685,16 @@ def test_build_pipeline_fails_permanently_when_the_table_is_gone() -> None:assert error_message_matches(str(raised.value), TrinoSource().get_non_retryable_errors())+def test_worker_node_communication_failures_are_classified_as_retryable() -> None:+ message = (+ "Encountered too many errors talking to a worker node. The node may have crashed or be "+ "under too much load. This is probably a transient issue, so please retry your query in a "+ "few minutes. (getting task status http://10.0.0.5:8080/v1/task/x - 38 failures)"+ )++ assert error_message_matches(message, TrinoSource().get_retryable_errors())++def test_build_pipeline_refuses_a_disallowed_host_before_connecting() -> None:with (patch(f"{_TRINO_MODULE}.connect_trino") as mock_connect,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.pyindex ab90568db42e..92296bf52c9d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py@@ -97,6 +97,11 @@ TRINO_COLUMN_NOT_FOUND_ERROR = ("A column this table syncs no longer exists in Trino. Refresh the schemas so PostHog picks up the new ""columns, then resync.")+# The Trino coordinator's own fixed wording when it gives up on a worker node after repeatedly+# failing to reach it mid-query. The coordinator already retried internally before raising this, so+# by the time it reaches us the worker has likely crashed or is overloaded — a vendor-side blip a+# fresh query normally clears, not a PostHog or customer problem.+TRINO_WORKER_NODE_ERROR = "too many errors talking to a worker node"# Every message `trino_error_to_message` produces for a recognized failure. The source lists them as# its own error patterns, so the API shows the message instead of a generic one.