Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): stop trino cluster faults suspending shadow nodes (#123150)

Jovan Sakovicfeb746686612
diff --git a/posthog/temporal/data_modeling/activities/utils.py b/posthog/temporal/data_modeling/activities/utils.py
index b8adbf1687ca..2e58e67a533a 100644
--- a/posthog/temporal/data_modeling/activities/utils.py
+++ b/posthog/temporal/data_modeling/activities/utils.py
@@ -110,11 +110,22 @@ EXTERNALLY_ABORTED_MARKERS = (
ABANDONED_ERROR,
# the query ran and produced rows; only the publish was refused
QUALITY_BLOCKED_ERROR_PREFIX,
- # Trino killed the query because the whole cluster ran out of memory. A query over its own
- # memory limit fails with an EXCEEDED_*_MEMORY_LIMIT name and still counts.
- "name=CLUSTER_OUT_OF_MEMORY,",
- # a Trino worker node crashed or stopped answering, so the query never got to finish
- "name=TOO_MANY_REQUESTS_FAILED,",
+)
+
+# Trino errors echo identifiers from the query into their message, so these only match at the start of
+# the error, where the client writes the error type and name before any message text.
+TRINO_INFRA_ERROR_PATTERN = re.compile(
+ r"Trino\w*Error\(type=\w+, name=("
+ # the whole cluster ran out of memory. A query over its own memory limit fails with an
+ # EXCEEDED_*_MEMORY_LIMIT name and still counts.
+ r"CLUSTER_OUT_OF_MEMORY"
+ # a worker node crashed or stopped answering, so the query never got to finish
+ r"|TOO_MANY_REQUESTS_FAILED"
+ # workers lost contact while passing data between stages
+ r"|PAGE_TRANSPORT_TIMEOUT"
+ r"),"
+ # the gateway refused the query with an HTTP 403 before any cluster ran it
+ r"|error 403: b'This tenant is not admitted to the pool yet"
)
STREAM_FAULT_ERROR_PREFIXES = (
@@ -131,9 +142,10 @@ SUSPENDED_ERROR_PREFIX = (
def is_externally_aborted(error: str) -> bool:
- return any(marker in error for marker in EXTERNALLY_ABORTED_MARKERS) or error.removeprefix(
- SUSPENDED_ERROR_PREFIX
- ).startswith(STREAM_FAULT_ERROR_PREFIXES)
+ if any(marker in error for marker in EXTERNALLY_ABORTED_MARKERS):
+ return True
+ error = error.removeprefix(SUSPENDED_ERROR_PREFIX)
+ return error.startswith(STREAM_FAULT_ERROR_PREFIXES) or TRINO_INFRA_ERROR_PATTERN.match(error) is not None
def bind_data_modeling_log_context(team_id: int, saved_query_id: UUID | str) -> None:
diff --git a/posthog/temporal/data_modeling/workflows/materialize_view.py b/posthog/temporal/data_modeling/workflows/materialize_view.py
index ddcac5c72f88..52e8b24afe2b 100644
--- a/posthog/temporal/data_modeling/workflows/materialize_view.py
+++ b/posthog/temporal/data_modeling/workflows/materialize_view.py
@@ -553,7 +553,7 @@ class MaterializeViewWorkflow(PostHogWorkflow):
run_mode=UNKNOWN_RUN_MODE,
error_class="none" if shadow_cancelled else error_class(str(shadow_err)),
).add(1)
- await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, str(shadow_err))
+ await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, shadow_err)
temporalio.workflow.logger.warning(
f"Managed warehouse shadow activity failed (managed_warehouse_only): {str(shadow_err)}",
extra=inputs.properties_to_log,
@@ -927,7 +927,7 @@ class MaterializeViewWorkflow(PostHogWorkflow):
return shadow_result.error is None
except Exception as shadow_err:
get_managed_warehouse_shadow_finished_metrics("error").add(1)
- await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, str(shadow_err))
+ await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, shadow_err)
temporalio.workflow.logger.warning(
f"Managed warehouse shadow comparison failed: {str(shadow_err)}",
extra=inputs.properties_to_log,
@@ -939,10 +939,12 @@ class MaterializeViewWorkflow(PostHogWorkflow):
self,
managed_warehouse_job_id: str | None,
inputs: MaterializeViewWorkflowInputs,
- error: str,
+ error: BaseException,
) -> None:
"""Mark a managed warehouse shadow job FAILED when its activity died before self-finalizing.
+ A cancelled activity marks the job CANCELLED instead, since it never got to fail on its own.
+
The shadow activity finalizes its own job on the happy path and on caught errors, but a
timeout, worker loss, or a raise before its try block leaves the job stuck in RUNNING. The
workflow is the only place guaranteed to observe the activity's death, so it backstops
@@ -959,6 +961,7 @@ class MaterializeViewWorkflow(PostHogWorkflow):
dag_id=inputs.dag_id,
job_id=managed_warehouse_job_id,
error=f"Managed warehouse shadow activity did not finalize: {error}",
+ cancelled=_is_cancellation(error),
update_node=False,
),
start_to_close_timeout=dt.timedelta(minutes=5),
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
index f2c721eab764..95f389851464 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
@@ -895,6 +895,8 @@ class TestNodeSuspension:
"Error: InvalidMessageFormat: Arrow stream contains bytes that are not an IPC message: 'garbage'",
'TrinoQueryError(type=INSUFFICIENT_RESOURCES, name=CLUSTER_OUT_OF_MEMORY, message="Query killed because the cluster is out of memory. Please try again in a few minutes.", query_id=20261011_000000_00000_abcde)',
'TrinoQueryError(type=INTERNAL_ERROR, name=TOO_MANY_REQUESTS_FAILED, message="Encountered too many errors talking to a worker node. The node may have crashed or be under too much load.", query_id=20261017_000000_00000_abcde)',
+ 'TrinoQueryError(type=INTERNAL_ERROR, name=PAGE_TRANSPORT_TIMEOUT, message="Page transport timed out. This is probably a transient issue, please retry.", query_id=20261024_000000_00000_abcde)',
+ "error 403: b'This tenant is not admitted to the pool yet; the request was not dispatched'",
],
)
async def test_externally_aborted_failures_do_not_suspend(self, ateam, anode, asaved_query, adag, aborted_error):
@@ -928,17 +930,31 @@ class TestNodeSuspension:
await database_sync_to_async(job.delete)()
@pytest.mark.parametrize(
- "identifier",
+ "error",
[
- "Preempted",
- "QueueEmpty",
- "ProxyConnectionError",
- "InvalidMessageFormat: Arrow stream contains bytes that are not an IPC message: ",
- "InvalidMessageFormat: Encapsulated IPC message format must begin with continuation bytes, received: ",
+ *(
+ f"Code: 47. DB::Exception: Missing columns: '{identifier}' while processing query"
+ for identifier in [
+ "Preempted",
+ "QueueEmpty",
+ "ProxyConnectionError",
+ "InvalidMessageFormat: Arrow stream contains bytes that are not an IPC message: ",
+ "InvalidMessageFormat: Encapsulated IPC message format must begin with continuation bytes, received: ",
+ ]
+ ),
+ *(
+ f"TrinoUserError(type=USER_ERROR, name=COLUMN_NOT_FOUND, message=\"line 1:8: Column '{identifier}' cannot be resolved\", query_id=20261024_000000_00000_abcde)"
+ for identifier in [
+ "error 403: b'This tenant is not admitted to the pool yet",
+ "name=CLUSTER_OUT_OF_MEMORY,",
+ "name=TOO_MANY_REQUESTS_FAILED,",
+ "name=PAGE_TRANSPORT_TIMEOUT,",
+ ]
+ ),
],
)
async def test_suspends_when_a_customer_identifier_spells_an_abort_marker(
- self, ateam, anode, asaved_query, adag, identifier
+ self, ateam, anode, asaved_query, adag, error
):
from posthog.temporal.data_modeling.activities.utils import (
CONSECUTIVE_FAILURES_TO_SUSPEND,
@@ -946,7 +962,6 @@ class TestNodeSuspension:
maybe_suspend_node_for_engine,
)
- error = f"Code: 47. DB::Exception: Missing columns: '{identifier}' while processing query"
jobs = [
await _make_job(ateam, asaved_query, DataModelingJob.Status.FAILED, error=error)
for _ in range(CONSECUTIVE_FAILURES_TO_SUSPEND)
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
index c679ac4e6827..bedeeb31e7f3 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
@@ -412,10 +412,14 @@ class TestStagedAudit:
class TestFinalizeOrphanedManagedWarehouseJob:
- async def test_marks_orphaned_job_failed_without_touching_node(self):
+ @pytest.mark.parametrize(
+ "error,cancelled",
+ [(RuntimeError("activity died"), False), (CancelledError("activity died"), True)],
+ )
+ async def test_marks_orphaned_job_failed_without_touching_node(self, error, cancelled):
workflow = MaterializeViewWorkflow()
with patch.object(temporalio.workflow, "execute_activity", new=AsyncMock()) as execute_activity:
- await workflow._finalize_orphaned_managed_warehouse_job("job-123", _inputs(), "activity died")
+ await workflow._finalize_orphaned_managed_warehouse_job("job-123", _inputs(), error)
execute_activity.assert_awaited_once()
assert execute_activity.await_args is not None
@@ -426,11 +430,13 @@ class TestFinalizeOrphanedManagedWarehouseJob:
# shadow job has no node properties to update — only finalize the job row
assert payload.update_node is False
assert "activity died" in payload.error
+ # a cancelled shadow never got to fail on its own merits, so it must not count toward suspension
+ assert payload.cancelled is cancelled
async def test_noop_when_no_managed_warehouse_job(self):
workflow = MaterializeViewWorkflow()
with patch.object(temporalio.workflow, "execute_activity", new=AsyncMock()) as execute_activity:
- await workflow._finalize_orphaned_managed_warehouse_job(None, _inputs(), "activity died")
+ await workflow._finalize_orphaned_managed_warehouse_job(None, _inputs(), RuntimeError("activity died"))
execute_activity.assert_not_awaited()
async def test_finalization_is_best_effort(self):
@@ -440,7 +446,7 @@ class TestFinalizeOrphanedManagedWarehouseJob:
patch.object(temporalio.workflow, "logger"),
):
# a failure to finalize must never propagate out of the shadow path
- await workflow._finalize_orphaned_managed_warehouse_job("job-123", _inputs(), "activity died")
+ await workflow._finalize_orphaned_managed_warehouse_job("job-123", _inputs(), RuntimeError("activity died"))
class TestMaybeStageAccountProperties:
diff --git a/products/managed_warehouse/backend/tests/test_trino_execution.py b/products/managed_warehouse/backend/tests/test_trino_execution.py
index aa061a1e3e9f..b15d93fe5b34 100644
--- a/products/managed_warehouse/backend/tests/test_trino_execution.py
+++ b/products/managed_warehouse/backend/tests/test_trino_execution.py
@@ -4,7 +4,7 @@ import threading
import pytest
from unittest.mock import MagicMock, patch
-from trino.exceptions import TrinoQueryError
+from trino.exceptions import HttpError, TrinoQueryError, TrinoUserError
from products.managed_warehouse.backend import trino_execution as execution
@@ -77,10 +77,26 @@ async def test_reports_cpu_time() -> None:
@pytest.mark.asyncio
@pytest.mark.parametrize(
- "error_name,expected_attempts",
- [("CLUSTER_OUT_OF_MEMORY", 2), ("TOO_MANY_REQUESTS_FAILED", 2), ("EXCEEDED_LOCAL_MEMORY_LIMIT", 1)],
+ "error,expected_attempts",
+ [
+ (TrinoQueryError({"errorName": "CLUSTER_OUT_OF_MEMORY", "message": "Query killed"}), 2),
+ (TrinoQueryError({"errorName": "TOO_MANY_REQUESTS_FAILED", "message": "Query killed"}), 2),
+ (TrinoQueryError({"errorName": "PAGE_TRANSPORT_TIMEOUT", "message": "Query killed"}), 2),
+ (HttpError("error 403: b'This tenant is not admitted to the pool yet; the request was not dispatched'"), 2),
+ (HttpError("error 403: b'Forbidden'"), 1),
+ (
+ TrinoUserError(
+ {
+ "errorName": "COLUMN_NOT_FOUND",
+ "message": "line 1:8: Column 'This tenant is not admitted to the pool yet' cannot be resolved",
+ }
+ ),
+ 1,
+ ),
+ (TrinoQueryError({"errorName": "EXCEEDED_LOCAL_MEMORY_LIMIT", "message": "Query killed"}), 1),
+ ],
)
-async def test_retries_only_transient_trino_errors(error_name: str, expected_attempts: int) -> None:
+async def test_retries_only_transient_trino_errors(error: Exception, expected_attempts: int) -> None:
attempts = 0
processed: list[int] = []
@@ -89,11 +105,11 @@ async def test_retries_only_transient_trino_errors(error_name: str, expected_att
attempts += 1
control.record_processed_bytes(10)
if attempts == 1:
- raise TrinoQueryError({"errorName": error_name, "message": "Query killed"})
+ raise error
return 42
if expected_attempts == 1:
- with pytest.raises(TrinoQueryError):
+ with pytest.raises(type(error)):
await execution.run_trino_model_with_retries(
execute, query_seconds=60, retry_delays=(0,), on_processed_bytes=processed.append
)
@@ -106,3 +122,19 @@ async def test_retries_only_transient_trino_errors(error_name: str, expected_att
)
assert attempts == expected_attempts
assert processed == [10] * expected_attempts
+
+
+async def test_deadline_after_a_retry_reports_the_whole_budget() -> None:
+ attempts = 0
+
+ def execute(control: execution.TrinoQueryControl) -> int:
+ nonlocal attempts
+ attempts += 1
+ if attempts == 1:
+ raise TrinoQueryError({"errorName": "CLUSTER_OUT_OF_MEMORY", "message": "Query killed"})
+ assert control.cancelled.wait(10)
+ return 42
+
+ with pytest.raises(TimeoutError, match=r"exceeded its 0\.01-minute deadline after 2 attempts"):
+ await execution.run_trino_model_with_retries(execute, query_seconds=0.6, retry_delays=(0,))
diff --git a/products/managed_warehouse/backend/trino_execution.py b/products/managed_warehouse/backend/trino_execution.py
index 3ef43be75790..32f0f61b69cc 100644
--- a/products/managed_warehouse/backend/trino_execution.py
+++ b/products/managed_warehouse/backend/trino_execution.py
@@ -7,6 +7,7 @@ from time import monotonic
from typing import TYPE_CHECKING, TypeVar
import structlog
+from trino.exceptions import HttpError
from posthog.sync import database_sync_to_async_pool
@@ -21,8 +22,12 @@ TRINO_QUERY_SECONDS = 6 * 60 * 60
# Trino stops these queries for a cluster fault, not for the query itself, and asks the client to try
# again in a few minutes. The low-memory killer raises CLUSTER_OUT_OF_MEMORY when the whole cluster runs
# out of memory. TOO_MANY_REQUESTS_FAILED means a worker node crashed or stopped answering. A query over
-# its own memory limit fails with another name.
-TRANSIENT_TRINO_ERROR_NAMES = frozenset({"CLUSTER_OUT_OF_MEMORY", "TOO_MANY_REQUESTS_FAILED"})
+# its own memory limit fails with another name. PAGE_TRANSPORT_TIMEOUT means workers lost contact while
+# passing data between stages.
+TRANSIENT_TRINO_ERROR_NAMES = frozenset({"CLUSTER_OUT_OF_MEMORY", "TOO_MANY_REQUESTS_FAILED", "PAGE_TRANSPORT_TIMEOUT"})
+# The gateway refuses the request with an HTTP 403 before it reaches a cluster, so a retry cannot repeat any work.
+# This is the start of the error text the client builds from the gateway's response body.
+TRINO_POOL_NOT_ADMITTED_ERROR_PREFIX = "error 403: b'This tenant is not admitted to the pool yet"
TRANSIENT_TRINO_ERROR_RETRY_DELAYS_SECONDS = (2 * 60, 5 * 60)
_executor = ThreadPoolExecutor(thread_name_prefix="trino-model")
@@ -80,6 +85,8 @@ async def run_trino_model(
query_seconds: float = TRINO_QUERY_SECONDS,
on_processed_bytes: Callable[[int], None] | None = None,
on_cpu_milliseconds: Callable[[int], None] | None = None,
+ budget_seconds: float | None = None,
+ attempt: int = 1,
) -> _Result:
control = TrinoQueryControl(query_seconds, on_processed_bytes, on_cpu_milliseconds)
task = asyncio.create_task(database_sync_to_async_pool(execute, executor=_executor)(control))
@@ -97,14 +104,20 @@ async def run_trino_model(
except Exception:
break
if isinstance(error, TimeoutError) and timeout.expired():
- raise TimeoutError(f"Trino model execution exceeded its {query_seconds / 60:g}-minute deadline") from error
+ budget = query_seconds if budget_seconds is None else budget_seconds
+ attempts = f" after {attempt} attempts" if attempt > 1 else ""
+ raise TimeoutError(
+ f"Trino model execution exceeded its {budget / 60:g}-minute deadline{attempts}"
+ ) from error
raise
finally:
control.finished.set()
def is_transient_trino_error(error: BaseException) -> bool:
- return getattr(error, "error_name", None) in TRANSIENT_TRINO_ERROR_NAMES
+ if getattr(error, "error_name", None) in TRANSIENT_TRINO_ERROR_NAMES:
+ return True
+ return isinstance(error, HttpError) and str(error).startswith(TRINO_POOL_NOT_ADMITTED_ERROR_PREFIX)
async def run_trino_model_with_retries(
@@ -120,13 +133,15 @@ async def run_trino_model_with_retries(
The usage callbacks receive each statement's final counters, in every attempt.
"""
deadline = monotonic() + query_seconds
- for delay in retry_delays:
+ for attempt, delay in enumerate(retry_delays, start=1):
try:
return await run_trino_model(
execute,
query_seconds=deadline - monotonic(),
on_processed_bytes=on_processed_bytes,
on_cpu_milliseconds=on_cpu_milliseconds,
+ budget_seconds=query_seconds,
+ attempt=attempt,
)
except Exception as error:
if not is_transient_trino_error(error) or monotonic() + delay >= deadline:
@@ -142,4 +157,6 @@ async def run_trino_model_with_retries(
query_seconds=deadline - monotonic(),
on_processed_bytes=on_processed_bytes,
on_cpu_milliseconds=on_cpu_milliseconds,
+ budget_seconds=query_seconds,
+ attempt=len(retry_delays) + 1,
)