:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): retry a momentary trino tenant pool admission delay (#122656)
Tom Owersd3803e4b479c
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.pyindex 045a2b649ace..b2e810af3e43 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py@@ -513,6 +513,45 @@ def test_discover_trino_schemas_fetches_columns_in_bounded_batches() -> None:assert all(len(call.args[1]) <= 101 for call in column_calls)+def test_discover_trino_schemas_retries_a_momentary_tenant_admission_delay() -> None:+ # The managed-Trino control plane returns this exact 403 when it mints a tenant's compute-pool+ # grant before the pool coordinator has caught up, right after a tenant is first provisioned.+ cursor = MagicMock()+ cursor.execute.side_effect = [+ HttpError("error 403: b'This tenant is not admitted to the pool yet; the request was not dispatched'"),+ None,+ None,+ ]+ cursor.fetchall.side_effect = [[("analytics", "events")], [("analytics", "events", "id", "bigint", "NO")]]++ with patch(f"{_TRINO_MODULE}.interruptible_wait") as mock_wait:+ discovered = discover_trino_schemas(cursor, _config())++ assert discovered == [+ DiscoveredTrinoTable(+ catalog="hive",+ schema="analytics",+ name="events",+ columns=(TrinoColumn(name="id", data_type="bigint", nullable=False),),+ )+ ]+ assert cursor.execute.call_count == 3+ mock_wait.assert_called_once()+++def test_discover_trino_schemas_gives_up_after_repeated_tenant_admission_delays() -> None:+ cursor = MagicMock()+ cursor.execute.side_effect = HttpError(+ "error 403: b'This tenant is not admitted to the pool yet; the request was not dispatched'"+ )++ with patch(f"{_TRINO_MODULE}.interruptible_wait") as mock_wait, pytest.raises(HttpError):+ discover_trino_schemas(cursor, _config())++ assert cursor.execute.call_count == 4+ assert mock_wait.call_count == 3++_TRINO_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.trino.trino"diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.pyindex f46f29356091..800a518e178c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py@@ -26,6 +26,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import DEFAULT_CHUNK_SIZEfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import incremental_type_to_initial_valuefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (+ interruptible_wait,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (ValidateDatabaseHostMixin,log_connection_open,@@ -164,6 +167,12 @@ TRINO_PERMANENT_SYNC_ERROR_MESSAGES = frozenset(TRINO_CLUSTER_ACCESS_DENIED_ERROR,})+# The managed-Trino control plane's own wording when it mints a tenant's compute-pool grant before+# the pool coordinator has finished admitting that tenant. The request was never dispatched, so a+# retry is safe, and the race clears within a few seconds once admission catches up.+TRINO_TENANT_NOT_ADMITTED_ERROR = "is not admitted to the pool yet"+_TENANT_ADMISSION_RETRY_ATTEMPTS = 4+_TENANT_ADMISSION_RETRY_BACKOFF_SECONDS = 1.0POSTHOG_MANAGED_TRINO_HOST_PATTERN = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.dw\.(?:dev|eu|us)\.postwh\.com")# Trino's OPA access control can issue one column-filter request per table, so keep# discovery batches comfortably below the HTTP client's outstanding-request limit.@@ -384,6 +393,19 @@ def discover_trino_schemas(return discover_trino_catalog(cursor, catalog=config.catalog, schema=config.schema, names=names)+def _execute_with_tenant_admission_retry(cursor: Cursor, query: str, parameters: list[object]) -> None:+ from trino.exceptions import HttpError # noqa: PLC0415 — keeps the optional driver off startup paths++ for attempt in range(1, _TENANT_ADMISSION_RETRY_ATTEMPTS + 1):+ try:+ cursor.execute(query, parameters)+ return+ except HttpError as exc:+ if attempt == _TENANT_ADMISSION_RETRY_ATTEMPTS or TRINO_TENANT_NOT_ADMITTED_ERROR not in str(exc).lower():+ raise+ interruptible_wait(_TENANT_ADMISSION_RETRY_BACKOFF_SECONDS * attempt)++def discover_trino_catalog(cursor: Cursor,*,@@ -403,7 +425,7 @@ def discover_trino_catalog(table_query += " AND table_schema = ?"parameters.append(schema)table_query += " ORDER BY table_schema, table_name"- cursor.execute(table_query, parameters)+ _execute_with_tenant_admission_retry(cursor, table_query, parameters)requested = set(names) if names is not None else Nonetable_names_by_schema: dict[str, list[str]] = {}@@ -429,7 +451,7 @@ def discover_trino_catalog(f"WHERE table_schema = ? AND table_name IN ({placeholders}) ""ORDER BY table_schema, table_name, ordinal_position")- cursor.execute(column_query, [schema_name, *table_name_batch])+ _execute_with_tenant_admission_retry(cursor, column_query, [schema_name, *table_name_batch])for row_schema, table_name, column_name, data_type, is_nullable in cursor.fetchall():tables.setdefault((str(row_schema), str(table_name)), []).append(TrinoColumn(