:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): treat trino spill exhaustion as retryable (#119331)
posthog[bot]266d6f596b1f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.pyindex 8a220a0ec217..4383802e23c6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py@@ -26,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_from products.warehouse_sources.backend.temporal.data_imports.sources.trino.trino import (TRINO_COLUMN_NOT_FOUND_ERROR,TRINO_KNOWN_ERROR_MESSAGES,+ TRINO_OUT_OF_SPILL_SPACE_ERROR,TRINO_QUERY_MEMORY_LIMIT_ERROR,TRINO_QUERY_NOT_FOUND_ERROR,TRINO_SCHEMA_NOT_FOUND_ERROR,@@ -292,7 +293,9 @@ class TrinoSource(SQLSource[TrinoSourceConfig], ValidateDatabaseHostMixin):}def get_retryable_errors(self) -> set[str]:- # Both are self-recovering vendor-side blips: the coordinator only raises the worker-node- # error after exhausting its own attempts to reach the worker, and a fresh attempt gets a- # new query ID, so a query the coordinator no longer holds ("Query not found") clears itself.- return {TRINO_WORKER_NODE_ERROR, TRINO_QUERY_NOT_FOUND_ERROR}+ # The worker-node error and "Query not found" are self-recovering vendor-side blips: the+ # coordinator only raises the former after exhausting its own attempts to reach the worker,+ # and a fresh attempt gets a new query ID, so a query the coordinator no longer holds clears+ # itself. Spill exhaustion depends on the workers' free disk at that moment, which frees up+ # as other queries finish, so a later attempt can succeed.+ return {TRINO_WORKER_NODE_ERROR, TRINO_QUERY_NOT_FOUND_ERROR, TRINO_OUT_OF_SPILL_SPACE_ERROR}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.pyindex 96c3e24d312e..045a2b649ace 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py@@ -827,13 +827,26 @@ def test_build_pipeline_fails_permanently_when_the_table_is_gone() -> None:assert error_message_matches(str(raised.value), TrinoSource().get_non_retryable_errors())-def test_worker_node_communication_failures_are_classified_as_retryable() -> None:- message = (+ "message",+ ["Encountered too many errors talking to a worker node. The node may have crashed or be ""under too much load. This is probably a transient issue, so please retry your query in a "- "few minutes. (getting task status http://10.0.0.5:8080/v1/task/x - 38 failures)"- )-+ "few minutes. (getting task status http://10.0.0.5:8080/v1/task/x - 38 failures)",+ str(+ TrinoQueryError(+ {+ "errorName": "OUT_OF_SPILL_SPACE",+ "errorType": "INSUFFICIENT_RESOURCES",+ "message": "No free or healthy space available for spill",+ },+ query_id="20260101_000000_00000_abcde",+ )+ ),+ ],+ ids=["worker_node", "out_of_spill_space"],+)+def test_remote_cluster_resource_failures_are_classified_as_retryable(message: str) -> None:assert error_message_matches(message, TrinoSource().get_retryable_errors())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.pyindex 8451f110f6ad..cd7a995d408e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py@@ -134,6 +134,9 @@ TRINO_WORKER_NODE_ERROR = "too many errors talking to a worker node"# `OperationalError`. A fresh query on the next sync attempt gets a new query ID, so this clears on# its own — not a PostHog or customer problem.TRINO_QUERY_NOT_FOUND_ERROR = "Query not found"+# Trino's error name when no worker has free or healthy spill disk left. The customer's cluster ran out+# of a resource it controls, so only the cluster owner can fix it, and a later query can succeed.+TRINO_OUT_OF_SPILL_SPACE_ERROR = "name=OUT_OF_SPILL_SPACE"# Every message `trino_error_to_message` produces for a recognized failure. The source lists them as# its own error patterns, so the API shows the message instead of a generic one.