Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(trino): stop reporting worker-node blips as exceptions (#118351)

Tom Owers8294182f4f28
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
index 26d96bec1c0a..9ad9bc5ef897 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
@@ -28,6 +28,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.trino.trin
TRINO_KNOWN_ERROR_MESSAGES,
TRINO_SCHEMA_NOT_FOUND_ERROR,
TRINO_TABLE_NOT_FOUND_ERROR,
+ TRINO_WORKER_NODE_ERROR,
TrinoImplementation,
TrinoSchemaDiscoveryError,
connect_trino,
@@ -283,3 +284,8 @@ class TrinoSource(SQLSource[TrinoSourceConfig], ValidateDatabaseHostMixin):
"name=COLUMN_NOT_FOUND": TRINO_COLUMN_NOT_FOUND_ERROR,
TrinoSchemaDiscoveryError.__name__: None,
}
+
+ def get_retryable_errors(self) -> set[str]:
+ # The coordinator only raises this after exhausting its own attempts to reach the worker,
+ # so it reaches us already a self-recovering vendor-side blip.
+ return {TRINO_WORKER_NODE_ERROR}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
index 46335d1aa7f4..52d7bf335858 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
@@ -685,6 +685,16 @@ def test_build_pipeline_fails_permanently_when_the_table_is_gone() -> None:
assert error_message_matches(str(raised.value), TrinoSource().get_non_retryable_errors())
+def test_worker_node_communication_failures_are_classified_as_retryable() -> None:
+ message = (
+ "Encountered too many errors talking to a worker node. The node may have crashed or be "
+ "under too much load. This is probably a transient issue, so please retry your query in a "
+ "few minutes. (getting task status http://10.0.0.5:8080/v1/task/x - 38 failures)"
+ )
+
+ assert error_message_matches(message, TrinoSource().get_retryable_errors())
+
+
def test_build_pipeline_refuses_a_disallowed_host_before_connecting() -> None:
with (
patch(f"{_TRINO_MODULE}.connect_trino") as mock_connect,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
index ab90568db42e..92296bf52c9d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
@@ -97,6 +97,11 @@ TRINO_COLUMN_NOT_FOUND_ERROR = (
"A column this table syncs no longer exists in Trino. Refresh the schemas so PostHog picks up the new "
"columns, then resync."
)
+# The Trino coordinator's own fixed wording when it gives up on a worker node after repeatedly
+# failing to reach it mid-query. The coordinator already retried internally before raising this, so
+# by the time it reaches us the worker has likely crashed or is overloaded — a vendor-side blip a
+# fresh query normally clears, not a PostHog or customer problem.
+TRINO_WORKER_NODE_ERROR = "too many errors talking to a worker node"
# Every message `trino_error_to_message` produces for a recognized failure. The source lists them as
# its own error patterns, so the API shows the message instead of a generic one.