Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): treat trino spill exhaustion as retryable (#119331)

posthog[bot]266d6f596b1f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
index 8a220a0ec217..4383802e23c6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
@@ -26,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
from products.warehouse_sources.backend.temporal.data_imports.sources.trino.trino import (
TRINO_COLUMN_NOT_FOUND_ERROR,
TRINO_KNOWN_ERROR_MESSAGES,
+ TRINO_OUT_OF_SPILL_SPACE_ERROR,
TRINO_QUERY_MEMORY_LIMIT_ERROR,
TRINO_QUERY_NOT_FOUND_ERROR,
TRINO_SCHEMA_NOT_FOUND_ERROR,
@@ -292,7 +293,9 @@ class TrinoSource(SQLSource[TrinoSourceConfig], ValidateDatabaseHostMixin):
}
def get_retryable_errors(self) -> set[str]:
- # Both are self-recovering vendor-side blips: the coordinator only raises the worker-node
- # error after exhausting its own attempts to reach the worker, and a fresh attempt gets a
- # new query ID, so a query the coordinator no longer holds ("Query not found") clears itself.
- return {TRINO_WORKER_NODE_ERROR, TRINO_QUERY_NOT_FOUND_ERROR}
+ # The worker-node error and "Query not found" are self-recovering vendor-side blips: the
+ # coordinator only raises the former after exhausting its own attempts to reach the worker,
+ # and a fresh attempt gets a new query ID, so a query the coordinator no longer holds clears
+ # itself. Spill exhaustion depends on the workers' free disk at that moment, which frees up
+ # as other queries finish, so a later attempt can succeed.
+ return {TRINO_WORKER_NODE_ERROR, TRINO_QUERY_NOT_FOUND_ERROR, TRINO_OUT_OF_SPILL_SPACE_ERROR}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
index 96c3e24d312e..045a2b649ace 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
@@ -827,13 +827,26 @@ def test_build_pipeline_fails_permanently_when_the_table_is_gone() -> None:
assert error_message_matches(str(raised.value), TrinoSource().get_non_retryable_errors())
-def test_worker_node_communication_failures_are_classified_as_retryable() -> None:
- message = (
+ "message",
+ [
"Encountered too many errors talking to a worker node. The node may have crashed or be "
"under too much load. This is probably a transient issue, so please retry your query in a "
- "few minutes. (getting task status http://10.0.0.5:8080/v1/task/x - 38 failures)"
- )
-
+ "few minutes. (getting task status http://10.0.0.5:8080/v1/task/x - 38 failures)",
+ str(
+ TrinoQueryError(
+ {
+ "errorName": "OUT_OF_SPILL_SPACE",
+ "errorType": "INSUFFICIENT_RESOURCES",
+ "message": "No free or healthy space available for spill",
+ },
+ query_id="20260101_000000_00000_abcde",
+ )
+ ),
+ ],
+ ids=["worker_node", "out_of_spill_space"],
+)
+def test_remote_cluster_resource_failures_are_classified_as_retryable(message: str) -> None:
assert error_message_matches(message, TrinoSource().get_retryable_errors())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
index 8451f110f6ad..cd7a995d408e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
@@ -134,6 +134,9 @@ TRINO_WORKER_NODE_ERROR = "too many errors talking to a worker node"
# `OperationalError`. A fresh query on the next sync attempt gets a new query ID, so this clears on
# its own — not a PostHog or customer problem.
TRINO_QUERY_NOT_FOUND_ERROR = "Query not found"
+# Trino's error name when no worker has free or healthy spill disk left. The customer's cluster ran out
+# of a resource it controls, so only the cluster owner can fix it, and a later query can succeed.
+TRINO_OUT_OF_SPILL_SPACE_ERROR = "name=OUT_OF_SPILL_SPACE"
# Every message `trino_error_to_message` produces for a recognized failure. The source lists them as
# its own error patterns, so the API shows the message instead of a generic one.