:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(trino): stop retrying a Trino sync that hits its own resource limits (#119328)
Tom Owersd2f7e1919c3b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.pyindex 5b5ee7801b9b..49a39e3a71e0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py@@ -26,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_from products.warehouse_sources.backend.temporal.data_imports.sources.trino.trino import (TRINO_COLUMN_NOT_FOUND_ERROR,TRINO_KNOWN_ERROR_MESSAGES,+ TRINO_QUERY_MEMORY_LIMIT_ERROR,TRINO_QUERY_NOT_FOUND_ERROR,TRINO_SCHEMA_NOT_FOUND_ERROR,TRINO_TABLE_NOT_FOUND_ERROR,@@ -283,6 +284,10 @@ class TrinoSource(SQLSource[TrinoSourceConfig], ValidateDatabaseHostMixin):"name=TABLE_NOT_FOUND": TRINO_TABLE_NOT_FOUND_ERROR,"name=SCHEMA_NOT_FOUND": TRINO_SCHEMA_NOT_FOUND_ERROR,"name=COLUMN_NOT_FOUND": TRINO_COLUMN_NOT_FOUND_ERROR,+ # Raised mid-sync by a query against the cluster, not schema discovery, so this matches+ # on the driver's own stable wording rather than an errorName. See the comment on the+ # constant for why retrying can't recover from it.+ "exceeded distributed total memory limit": TRINO_QUERY_MEMORY_LIMIT_ERROR,TrinoSchemaDiscoveryError.__name__: None,}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.pyindex 10dfc8a8672a..d6bea32b057a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py@@ -11,7 +11,7 @@ from unittest.mock import MagicMock, patchimport pyarrow as pafrom requests.exceptions import ConnectTimeout, ReadTimeout-from trino.exceptions import HttpError, TrinoConnectionError, TrinoExternalError, TrinoUserError+from trino.exceptions import HttpError, TrinoConnectionError, TrinoExternalError, TrinoQueryError, TrinoUserErrorfrom trino.types import NamedRowTuplefrom products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (@@ -781,6 +781,22 @@ def test_get_schemas_advertises_incremental_fields_per_table() -> None:)+def _trino_query_error(message: str) -> TrinoQueryError:+ # Resource-exhaustion failures carry an errorType the client doesn't special-case+ # ("INSUFFICIENT_RESOURCES", neither "EXTERNAL" nor "USER_ERROR"), so the driver raises the+ # base TrinoQueryError rather than a USER/EXTERNAL subclass.+ return TrinoQueryError(+ {+ "message": message,+ "errorName": "GENERIC_INSUFFICIENT_RESOURCES",+ "errorCode": 65540,+ "errorType": "INSUFFICIENT_RESOURCES",+ "failureInfo": {},+ },+ "20260902_120000_00000_abcde",+ )++def _trino_user_error(name: str, message: str) -> TrinoUserError:return TrinoUserError({"message": message, "errorName": name, "errorCode": 1, "errorType": "USER_ERROR", "failureInfo": {}},@@ -797,8 +813,21 @@ def _trino_user_error(name: str, message: str) -> TrinoUserError:(_trino_user_error("COLUMN_NOT_FOUND", "line 1:8: Column 'gone' cannot be resolved"), True),(_trino_external_error("Failed to query OPA backend"), False),(TrinoConnectionError("failed to execute: ('Connection aborted.', ConnectionResetError(104))"), False),+ (_trino_query_error("Query exceeded distributed total memory limit of 2.64TB"), True),+ # Deliberately retryable, not a regression: this depends on a worker's free disk at that+ # moment, shared by every query Trino schedules there, so it can clear on its own.+ (_trino_query_error("No free or healthy space available for spill"), False),+ ],+ ids=[+ "access_denied",+ "unauthorized",+ "table_not_found",+ "column_not_found",+ "opa_unavailable",+ "connection_reset",+ "query_memory_limit",+ "spill_space_exhausted",],- ids=["access_denied", "unauthorized", "table_not_found", "column_not_found", "opa_unavailable", "connection_reset"],)def test_sync_failures_are_classified_for_retry(error: Exception, is_permanent: bool) -> None:with pytest.raises(Exception) as raised:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.pyindex 1b4c5ee6f71a..3f8e1d046137 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py@@ -97,6 +97,23 @@ TRINO_COLUMN_NOT_FOUND_ERROR = ("A column this table syncs no longer exists in Trino. Refresh the schemas so PostHog picks up the new ""columns, then resync.")+# The incremental read orders by its watermark column so a resumed sync never skips a row, which+# can force Trino to sort the whole matching result set instead of pushing the sort down to the+# connector. A table (or incremental window) larger than the cluster's fixed per-query memory limit+# always hits it again: the limit doesn't depend on what else is running, so a retry replays the+# identical query against the same ceiling. Same capacity-limit shape as ClickHouse's Code: 241 and+# Postgres/MySQL's out-of-memory messages: stop and tell them to act.+#+# A spill-space failure ("No free or healthy space available for spill") is deliberately not+# classified here. It reflects a worker's free disk at that moment, shared by every query Trino+# schedules there, so it can clear once another query finishes and frees its spill files - a retry+# can genuinely recover without the customer doing anything.+TRINO_QUERY_MEMORY_LIMIT_ERROR = (+ "Trino ran out of query memory while reading this table (it hit the cluster's distributed total "+ "memory limit for a single query). This usually means the table, or the current incremental window, "+ "is too large for the cluster's memory limits. Raise Trino's query memory limits, or sync a smaller "+ "table, fewer columns, or a narrower incremental window, then resync."+)# The Trino coordinator's own fixed wording when it gives up on a worker node after repeatedly# failing to reach it mid-query. The coordinator already retried internally before raising this, so# by the time it reaches us the worker has likely crashed or is overloaded — a vendor-side blip a