:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(trino): explain a timeout while listing tables (#118395)
Tom Owersfe2777f1f14f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.pyindex 993d0f9ffba4..46335d1aa7f4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py@@ -10,6 +10,7 @@ import pytestfrom unittest.mock import MagicMock, patchimport pyarrow as pa+from requests.exceptions import ConnectTimeout, ReadTimeoutfrom trino.exceptions import HttpError, TrinoConnectionError, TrinoExternalError, TrinoUserErrorfrom trino.types import NamedRowTuple@@ -27,7 +28,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.trino.trinTRINO_ACCESS_CONTROL_UNAVAILABLE_ERROR,TRINO_ACCESS_DENIED_ERROR,TRINO_AUTHENTICATION_ERROR,+ TRINO_CONNECT_TIMEOUT_ERROR,TRINO_CREDENTIALS_REQUIRE_TLS_VERIFICATION_ERROR,+ TRINO_DISCOVERY_TIMEOUT_ERROR,+ TRINO_READ_TIMEOUT_ERROR,DiscoveredTrinoTable,TrinoColumn,TrinoImplementation,@@ -199,6 +203,47 @@ def test_trino_error_to_message_explains_access_failures(raw_error: str, expecteassert trino_error_to_message(RuntimeError(raw_error)) == expected+ ("timeout", "expected"),+ [+ (+ ConnectTimeout("HTTPSConnectionPool(host='trino.example.com', port=443): Max retries exceeded"),+ TRINO_CONNECT_TIMEOUT_ERROR,+ ),+ (+ ReadTimeout("HTTPSConnectionPool(host='trino.example.com', port=443): Read timed out."),+ TRINO_READ_TIMEOUT_ERROR,+ ),+ ],+ ids=["connect_timeout", "read_timeout"],+)+def test_validate_credentials_hides_the_host_when_the_driver_times_out(timeout: Exception, expected: str) -> None:+ with patch("trino.client.TrinoRequest.post", side_effect=timeout):+ assert TrinoSource().validate_credentials(_config(), team_id=1) == (False, expected)+++ ("timeout", "expected"),+ [+ (ReadTimeout("Read timed out. (read timeout=60)"), TRINO_DISCOVERY_TIMEOUT_ERROR),+ (ConnectTimeout("Max retries exceeded"), TRINO_CONNECT_TIMEOUT_ERROR),+ ],+ ids=["read_timeout", "connect_timeout"],+)+def test_get_schemas_explains_a_timeout_the_driver_wrapped(timeout: Exception, expected: str) -> None:+ source = TrinoSource()++ with (+ patch("trino.client.TrinoRequest.post", side_effect=timeout),+ pytest.raises(TrinoSchemaDiscoveryError) as raised,+ ):+ source.get_schemas(_config(), team_id=1)++ message, is_expected_source_error = _classify_refresh_schemas_error(source, raised.value)+ assert message == expected+ assert is_expected_source_error is True++def _trino_external_error(message: str) -> TrinoExternalError:return TrinoExternalError({diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.pyindex d47601d66253..ab90568db42e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py@@ -9,7 +9,7 @@ from itertools import batchedfrom typing import TYPE_CHECKING, Anyimport pyarrow as pa-from requests.exceptions import RequestException+from requests.exceptions import ConnectTimeout, RequestException, Timeoutfrom posthog.dataclasses import frozen@@ -75,6 +75,17 @@ TRINO_TLS_CERTIFICATE_ERROR = ("Check the certificate or turn off certificate verification.")TRINO_CONNECTION_ERROR = "PostHog could not connect to Trino."+TRINO_CONNECT_TIMEOUT_ERROR = (+ "Connection to Trino timed out. Check the host and port, and that PostHog's egress IP addresses are "+ "allowed through your firewall."+)+TRINO_READ_TIMEOUT_ERROR = (+ "Trino didn't answer in time. Check that your Trino cluster isn't overloaded, then try again."+)+TRINO_DISCOVERY_TIMEOUT_ERROR = (+ "Trino took too long to list your tables. If you left the schema blank, enter one so PostHog lists "+ "fewer tables, then try again."+)TRINO_TABLE_NOT_FOUND_ERROR = ("Trino could not find this table, or this user can no longer see its columns. ""Check that the table still exists and that the user can read it, then resync."@@ -97,6 +108,9 @@ TRINO_KNOWN_ERROR_MESSAGES = (TRINO_ACCESS_CONTROL_UNAVAILABLE_ERROR,TRINO_TLS_CERTIFICATE_ERROR,TRINO_CONNECTION_ERROR,+ TRINO_CONNECT_TIMEOUT_ERROR,+ TRINO_READ_TIMEOUT_ERROR,+ TRINO_DISCOVERY_TIMEOUT_ERROR,)# The subset of those failures that a retry of an import sync cannot fix. A broken connection, an# unreachable access control service and a TLS failure can all clear on their own, so a sync retries them.@@ -150,6 +164,17 @@ class TrinoConfigurationError(ValueError):"""A connection setting Trino cannot be reached with, found before PostHog opens the connection."""+def _timeout_in_chain(error: BaseException) -> Timeout | None:+ # The driver re-raises a requests timeout as `TrinoConnectionError` inside its `except` block,+ # so only `__context__` still carries the original.+ current: BaseException | None = error+ while current is not None:+ if isinstance(current, Timeout):+ return current+ current = current.__cause__ or current.__context__+ return None++@contextmanagerdef trino_failures_as_discovery_error() -> Iterator[None]:"""Word a failed trip to Trino for the person who connects the source.@@ -161,6 +186,10 @@ def trino_failures_as_discovery_error() -> Iterator[None]:try:yieldexcept (Error, HttpError, RequestException, TrinoConfigurationError) as exc:+ # Listing every schema in a large catalog is the slow part of discovery, so a schema narrows it.+ timeout = _timeout_in_chain(exc)+ if timeout is not None and not isinstance(timeout, ConnectTimeout):+ raise TrinoSchemaDiscoveryError(TRINO_DISCOVERY_TIMEOUT_ERROR) from excraise TrinoSchemaDiscoveryError(trino_error_to_message(exc)) from exc@@ -169,6 +198,12 @@ def trino_error_to_message(error: Exception) -> str:lowered = message.lower()if message == TRINO_CREDENTIALS_REQUIRE_TLS_VERIFICATION_ERROR:return message+ # The raw requests text names the host, so it never reaches the user.+ timeout = _timeout_in_chain(error)+ if isinstance(timeout, ConnectTimeout):+ return TRINO_CONNECT_TIMEOUT_ERROR+ if timeout is not None:+ return TRINO_READ_TIMEOUT_ERROR# Match the driver's HTTP 401 form ("error 401: ..."), not a bare "401": a Trino query error's# text carries a query ID whose time part can hold "401" and would derail this classification.if "authentication" in lowered or "unauthorized" in lowered or "error 401" in lowered: