Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(trino): stop retrying a Trino sync that hits its own resource limits (#119328)

Tom Owersd2f7e1919c3b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
index 5b5ee7801b9b..49a39e3a71e0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
@@ -26,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
from products.warehouse_sources.backend.temporal.data_imports.sources.trino.trino import (
TRINO_COLUMN_NOT_FOUND_ERROR,
TRINO_KNOWN_ERROR_MESSAGES,
+ TRINO_QUERY_MEMORY_LIMIT_ERROR,
TRINO_QUERY_NOT_FOUND_ERROR,
TRINO_SCHEMA_NOT_FOUND_ERROR,
TRINO_TABLE_NOT_FOUND_ERROR,
@@ -283,6 +284,10 @@ class TrinoSource(SQLSource[TrinoSourceConfig], ValidateDatabaseHostMixin):
"name=TABLE_NOT_FOUND": TRINO_TABLE_NOT_FOUND_ERROR,
"name=SCHEMA_NOT_FOUND": TRINO_SCHEMA_NOT_FOUND_ERROR,
"name=COLUMN_NOT_FOUND": TRINO_COLUMN_NOT_FOUND_ERROR,
+ # Raised mid-sync by a query against the cluster, not schema discovery, so this matches
+ # on the driver's own stable wording rather than an errorName. See the comment on the
+ # constant for why retrying can't recover from it.
+ "exceeded distributed total memory limit": TRINO_QUERY_MEMORY_LIMIT_ERROR,
TrinoSchemaDiscoveryError.__name__: None,
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
index 10dfc8a8672a..d6bea32b057a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
@@ -11,7 +11,7 @@ from unittest.mock import MagicMock, patch
import pyarrow as pa
from requests.exceptions import ConnectTimeout, ReadTimeout
-from trino.exceptions import HttpError, TrinoConnectionError, TrinoExternalError, TrinoUserError
+from trino.exceptions import HttpError, TrinoConnectionError, TrinoExternalError, TrinoQueryError, TrinoUserError
from trino.types import NamedRowTuple
from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (
@@ -781,6 +781,22 @@ def test_get_schemas_advertises_incremental_fields_per_table() -> None:
)
+def _trino_query_error(message: str) -> TrinoQueryError:
+ # Resource-exhaustion failures carry an errorType the client doesn't special-case
+ # ("INSUFFICIENT_RESOURCES", neither "EXTERNAL" nor "USER_ERROR"), so the driver raises the
+ # base TrinoQueryError rather than a USER/EXTERNAL subclass.
+ return TrinoQueryError(
+ {
+ "message": message,
+ "errorName": "GENERIC_INSUFFICIENT_RESOURCES",
+ "errorCode": 65540,
+ "errorType": "INSUFFICIENT_RESOURCES",
+ "failureInfo": {},
+ },
+ "20260902_120000_00000_abcde",
+ )
+
+
def _trino_user_error(name: str, message: str) -> TrinoUserError:
return TrinoUserError(
{"message": message, "errorName": name, "errorCode": 1, "errorType": "USER_ERROR", "failureInfo": {}},
@@ -797,8 +813,21 @@ def _trino_user_error(name: str, message: str) -> TrinoUserError:
(_trino_user_error("COLUMN_NOT_FOUND", "line 1:8: Column 'gone' cannot be resolved"), True),
(_trino_external_error("Failed to query OPA backend"), False),
(TrinoConnectionError("failed to execute: ('Connection aborted.', ConnectionResetError(104))"), False),
+ (_trino_query_error("Query exceeded distributed total memory limit of 2.64TB"), True),
+ # Deliberately retryable, not a regression: this depends on a worker's free disk at that
+ # moment, shared by every query Trino schedules there, so it can clear on its own.
+ (_trino_query_error("No free or healthy space available for spill"), False),
+ ],
+ ids=[
+ "access_denied",
+ "unauthorized",
+ "table_not_found",
+ "column_not_found",
+ "opa_unavailable",
+ "connection_reset",
+ "query_memory_limit",
+ "spill_space_exhausted",
],
- ids=["access_denied", "unauthorized", "table_not_found", "column_not_found", "opa_unavailable", "connection_reset"],
)
def test_sync_failures_are_classified_for_retry(error: Exception, is_permanent: bool) -> None:
with pytest.raises(Exception) as raised:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
index 1b4c5ee6f71a..3f8e1d046137 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
@@ -97,6 +97,23 @@ TRINO_COLUMN_NOT_FOUND_ERROR = (
"A column this table syncs no longer exists in Trino. Refresh the schemas so PostHog picks up the new "
"columns, then resync."
)
+# The incremental read orders by its watermark column so a resumed sync never skips a row, which
+# can force Trino to sort the whole matching result set instead of pushing the sort down to the
+# connector. A table (or incremental window) larger than the cluster's fixed per-query memory limit
+# always hits it again: the limit doesn't depend on what else is running, so a retry replays the
+# identical query against the same ceiling. Same capacity-limit shape as ClickHouse's Code: 241 and
+# Postgres/MySQL's out-of-memory messages: stop and tell them to act.
+#
+# A spill-space failure ("No free or healthy space available for spill") is deliberately not
+# classified here. It reflects a worker's free disk at that moment, shared by every query Trino
+# schedules there, so it can clear once another query finishes and frees its spill files - a retry
+# can genuinely recover without the customer doing anything.
+TRINO_QUERY_MEMORY_LIMIT_ERROR = (
+ "Trino ran out of query memory while reading this table (it hit the cluster's distributed total "
+ "memory limit for a single query). This usually means the table, or the current incremental window, "
+ "is too large for the cluster's memory limits. Raise Trino's query memory limits, or sync a smaller "
+ "table, fewer columns, or a narrower incremental window, then resync."
+)
# The Trino coordinator's own fixed wording when it gives up on a worker node after repeatedly
# failing to reach it mid-query. The coordinator already retried internally before raising this, so
# by the time it reaches us the worker has likely crashed or is overloaded — a vendor-side blip a