Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(trino): explain a timeout while listing tables (#118395)

Tom Owersfe2777f1f14f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
index 993d0f9ffba4..46335d1aa7f4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
@@ -10,6 +10,7 @@ import pytest
from unittest.mock import MagicMock, patch
import pyarrow as pa
+from requests.exceptions import ConnectTimeout, ReadTimeout
from trino.exceptions import HttpError, TrinoConnectionError, TrinoExternalError, TrinoUserError
from trino.types import NamedRowTuple
@@ -27,7 +28,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.trino.trin
TRINO_ACCESS_CONTROL_UNAVAILABLE_ERROR,
TRINO_ACCESS_DENIED_ERROR,
TRINO_AUTHENTICATION_ERROR,
+ TRINO_CONNECT_TIMEOUT_ERROR,
TRINO_CREDENTIALS_REQUIRE_TLS_VERIFICATION_ERROR,
+ TRINO_DISCOVERY_TIMEOUT_ERROR,
+ TRINO_READ_TIMEOUT_ERROR,
DiscoveredTrinoTable,
TrinoColumn,
TrinoImplementation,
@@ -199,6 +203,47 @@ def test_trino_error_to_message_explains_access_failures(raw_error: str, expecte
assert trino_error_to_message(RuntimeError(raw_error)) == expected
+ ("timeout", "expected"),
+ [
+ (
+ ConnectTimeout("HTTPSConnectionPool(host='trino.example.com', port=443): Max retries exceeded"),
+ TRINO_CONNECT_TIMEOUT_ERROR,
+ ),
+ (
+ ReadTimeout("HTTPSConnectionPool(host='trino.example.com', port=443): Read timed out."),
+ TRINO_READ_TIMEOUT_ERROR,
+ ),
+ ],
+ ids=["connect_timeout", "read_timeout"],
+)
+def test_validate_credentials_hides_the_host_when_the_driver_times_out(timeout: Exception, expected: str) -> None:
+ with patch("trino.client.TrinoRequest.post", side_effect=timeout):
+ assert TrinoSource().validate_credentials(_config(), team_id=1) == (False, expected)
+
+
+ ("timeout", "expected"),
+ [
+ (ReadTimeout("Read timed out. (read timeout=60)"), TRINO_DISCOVERY_TIMEOUT_ERROR),
+ (ConnectTimeout("Max retries exceeded"), TRINO_CONNECT_TIMEOUT_ERROR),
+ ],
+ ids=["read_timeout", "connect_timeout"],
+)
+def test_get_schemas_explains_a_timeout_the_driver_wrapped(timeout: Exception, expected: str) -> None:
+ source = TrinoSource()
+
+ with (
+ patch("trino.client.TrinoRequest.post", side_effect=timeout),
+ pytest.raises(TrinoSchemaDiscoveryError) as raised,
+ ):
+ source.get_schemas(_config(), team_id=1)
+
+ message, is_expected_source_error = _classify_refresh_schemas_error(source, raised.value)
+ assert message == expected
+ assert is_expected_source_error is True
+
+
def _trino_external_error(message: str) -> TrinoExternalError:
return TrinoExternalError(
{
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
index d47601d66253..ab90568db42e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
@@ -9,7 +9,7 @@ from itertools import batched
from typing import TYPE_CHECKING, Any
import pyarrow as pa
-from requests.exceptions import RequestException
+from requests.exceptions import ConnectTimeout, RequestException, Timeout
from posthog.dataclasses import frozen
@@ -75,6 +75,17 @@ TRINO_TLS_CERTIFICATE_ERROR = (
"Check the certificate or turn off certificate verification."
)
TRINO_CONNECTION_ERROR = "PostHog could not connect to Trino."
+TRINO_CONNECT_TIMEOUT_ERROR = (
+ "Connection to Trino timed out. Check the host and port, and that PostHog's egress IP addresses are "
+ "allowed through your firewall."
+)
+TRINO_READ_TIMEOUT_ERROR = (
+ "Trino didn't answer in time. Check that your Trino cluster isn't overloaded, then try again."
+)
+TRINO_DISCOVERY_TIMEOUT_ERROR = (
+ "Trino took too long to list your tables. If you left the schema blank, enter one so PostHog lists "
+ "fewer tables, then try again."
+)
TRINO_TABLE_NOT_FOUND_ERROR = (
"Trino could not find this table, or this user can no longer see its columns. "
"Check that the table still exists and that the user can read it, then resync."
@@ -97,6 +108,9 @@ TRINO_KNOWN_ERROR_MESSAGES = (
TRINO_ACCESS_CONTROL_UNAVAILABLE_ERROR,
TRINO_TLS_CERTIFICATE_ERROR,
TRINO_CONNECTION_ERROR,
+ TRINO_CONNECT_TIMEOUT_ERROR,
+ TRINO_READ_TIMEOUT_ERROR,
+ TRINO_DISCOVERY_TIMEOUT_ERROR,
)
# The subset of those failures that a retry of an import sync cannot fix. A broken connection, an
# unreachable access control service and a TLS failure can all clear on their own, so a sync retries them.
@@ -150,6 +164,17 @@ class TrinoConfigurationError(ValueError):
"""A connection setting Trino cannot be reached with, found before PostHog opens the connection."""
+def _timeout_in_chain(error: BaseException) -> Timeout | None:
+ # The driver re-raises a requests timeout as `TrinoConnectionError` inside its `except` block,
+ # so only `__context__` still carries the original.
+ current: BaseException | None = error
+ while current is not None:
+ if isinstance(current, Timeout):
+ return current
+ current = current.__cause__ or current.__context__
+ return None
+
+
@contextmanager
def trino_failures_as_discovery_error() -> Iterator[None]:
"""Word a failed trip to Trino for the person who connects the source.
@@ -161,6 +186,10 @@ def trino_failures_as_discovery_error() -> Iterator[None]:
try:
yield
except (Error, HttpError, RequestException, TrinoConfigurationError) as exc:
+ # Listing every schema in a large catalog is the slow part of discovery, so a schema narrows it.
+ timeout = _timeout_in_chain(exc)
+ if timeout is not None and not isinstance(timeout, ConnectTimeout):
+ raise TrinoSchemaDiscoveryError(TRINO_DISCOVERY_TIMEOUT_ERROR) from exc
raise TrinoSchemaDiscoveryError(trino_error_to_message(exc)) from exc
@@ -169,6 +198,12 @@ def trino_error_to_message(error: Exception) -> str:
lowered = message.lower()
if message == TRINO_CREDENTIALS_REQUIRE_TLS_VERIFICATION_ERROR:
return message
+ # The raw requests text names the host, so it never reaches the user.
+ timeout = _timeout_in_chain(error)
+ if isinstance(timeout, ConnectTimeout):
+ return TRINO_CONNECT_TIMEOUT_ERROR
+ if timeout is not None:
+ return TRINO_READ_TIMEOUT_ERROR
# Match the driver's HTTP 401 form ("error 401: ..."), not a bare "401": a Trino query error's
# text carries a query ID whose time part can hold "401" and would derail this classification.
if "authentication" in lowered or "unauthorized" in lowered or "error 401" in lowered: