:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): stop trino cluster faults suspending shadow nodes (#123150)
Jovan Sakovicfeb746686612
diff --git a/posthog/temporal/data_modeling/activities/utils.py b/posthog/temporal/data_modeling/activities/utils.pyindex b8adbf1687ca..2e58e67a533a 100644--- a/posthog/temporal/data_modeling/activities/utils.py+++ b/posthog/temporal/data_modeling/activities/utils.py@@ -110,11 +110,22 @@ EXTERNALLY_ABORTED_MARKERS = (ABANDONED_ERROR,# the query ran and produced rows; only the publish was refusedQUALITY_BLOCKED_ERROR_PREFIX,- # Trino killed the query because the whole cluster ran out of memory. A query over its own- # memory limit fails with an EXCEEDED_*_MEMORY_LIMIT name and still counts.- "name=CLUSTER_OUT_OF_MEMORY,",- # a Trino worker node crashed or stopped answering, so the query never got to finish- "name=TOO_MANY_REQUESTS_FAILED,",+)++# Trino errors echo identifiers from the query into their message, so these only match at the start of+# the error, where the client writes the error type and name before any message text.+TRINO_INFRA_ERROR_PATTERN = re.compile(+ r"Trino\w*Error\(type=\w+, name=("+ # the whole cluster ran out of memory. A query over its own memory limit fails with an+ # EXCEEDED_*_MEMORY_LIMIT name and still counts.+ r"CLUSTER_OUT_OF_MEMORY"+ # a worker node crashed or stopped answering, so the query never got to finish+ r"|TOO_MANY_REQUESTS_FAILED"+ # workers lost contact while passing data between stages+ r"|PAGE_TRANSPORT_TIMEOUT"+ r"),"+ # the gateway refused the query with an HTTP 403 before any cluster ran it+ r"|error 403: b'This tenant is not admitted to the pool yet")STREAM_FAULT_ERROR_PREFIXES = (@@ -131,9 +142,10 @@ SUSPENDED_ERROR_PREFIX = (def is_externally_aborted(error: str) -> bool:- return any(marker in error for marker in EXTERNALLY_ABORTED_MARKERS) or error.removeprefix(- SUSPENDED_ERROR_PREFIX- ).startswith(STREAM_FAULT_ERROR_PREFIXES)+ if any(marker in error for marker in EXTERNALLY_ABORTED_MARKERS):+ return True+ error = error.removeprefix(SUSPENDED_ERROR_PREFIX)+ return error.startswith(STREAM_FAULT_ERROR_PREFIXES) or TRINO_INFRA_ERROR_PATTERN.match(error) is not Nonedef bind_data_modeling_log_context(team_id: int, saved_query_id: UUID | str) -> None:diff --git a/posthog/temporal/data_modeling/workflows/materialize_view.py b/posthog/temporal/data_modeling/workflows/materialize_view.pyindex ddcac5c72f88..52e8b24afe2b 100644--- a/posthog/temporal/data_modeling/workflows/materialize_view.py+++ b/posthog/temporal/data_modeling/workflows/materialize_view.py@@ -553,7 +553,7 @@ class MaterializeViewWorkflow(PostHogWorkflow):run_mode=UNKNOWN_RUN_MODE,error_class="none" if shadow_cancelled else error_class(str(shadow_err)),).add(1)- await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, str(shadow_err))+ await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, shadow_err)temporalio.workflow.logger.warning(f"Managed warehouse shadow activity failed (managed_warehouse_only): {str(shadow_err)}",extra=inputs.properties_to_log,@@ -927,7 +927,7 @@ class MaterializeViewWorkflow(PostHogWorkflow):return shadow_result.error is Noneexcept Exception as shadow_err:get_managed_warehouse_shadow_finished_metrics("error").add(1)- await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, str(shadow_err))+ await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, shadow_err)temporalio.workflow.logger.warning(f"Managed warehouse shadow comparison failed: {str(shadow_err)}",extra=inputs.properties_to_log,@@ -939,10 +939,12 @@ class MaterializeViewWorkflow(PostHogWorkflow):self,managed_warehouse_job_id: str | None,inputs: MaterializeViewWorkflowInputs,- error: str,+ error: BaseException,) -> None:"""Mark a managed warehouse shadow job FAILED when its activity died before self-finalizing.+ A cancelled activity marks the job CANCELLED instead, since it never got to fail on its own.+The shadow activity finalizes its own job on the happy path and on caught errors, but atimeout, worker loss, or a raise before its try block leaves the job stuck in RUNNING. Theworkflow is the only place guaranteed to observe the activity's death, so it backstops@@ -959,6 +961,7 @@ class MaterializeViewWorkflow(PostHogWorkflow):dag_id=inputs.dag_id,job_id=managed_warehouse_job_id,error=f"Managed warehouse shadow activity did not finalize: {error}",+ cancelled=_is_cancellation(error),update_node=False,),start_to_close_timeout=dt.timedelta(minutes=5),diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.pyindex f2c721eab764..95f389851464 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py@@ -895,6 +895,8 @@ class TestNodeSuspension:"Error: InvalidMessageFormat: Arrow stream contains bytes that are not an IPC message: 'garbage'",'TrinoQueryError(type=INSUFFICIENT_RESOURCES, name=CLUSTER_OUT_OF_MEMORY, message="Query killed because the cluster is out of memory. Please try again in a few minutes.", query_id=20261011_000000_00000_abcde)','TrinoQueryError(type=INTERNAL_ERROR, name=TOO_MANY_REQUESTS_FAILED, message="Encountered too many errors talking to a worker node. The node may have crashed or be under too much load.", query_id=20261017_000000_00000_abcde)',+ 'TrinoQueryError(type=INTERNAL_ERROR, name=PAGE_TRANSPORT_TIMEOUT, message="Page transport timed out. This is probably a transient issue, please retry.", query_id=20261024_000000_00000_abcde)',+ "error 403: b'This tenant is not admitted to the pool yet; the request was not dispatched'",],)async def test_externally_aborted_failures_do_not_suspend(self, ateam, anode, asaved_query, adag, aborted_error):@@ -928,17 +930,31 @@ class TestNodeSuspension:await database_sync_to_async(job.delete)()@pytest.mark.parametrize(- "identifier",+ "error",[- "Preempted",- "QueueEmpty",- "ProxyConnectionError",- "InvalidMessageFormat: Arrow stream contains bytes that are not an IPC message: ",- "InvalidMessageFormat: Encapsulated IPC message format must begin with continuation bytes, received: ",+ *(+ f"Code: 47. DB::Exception: Missing columns: '{identifier}' while processing query"+ for identifier in [+ "Preempted",+ "QueueEmpty",+ "ProxyConnectionError",+ "InvalidMessageFormat: Arrow stream contains bytes that are not an IPC message: ",+ "InvalidMessageFormat: Encapsulated IPC message format must begin with continuation bytes, received: ",+ ]+ ),+ *(+ f"TrinoUserError(type=USER_ERROR, name=COLUMN_NOT_FOUND, message=\"line 1:8: Column '{identifier}' cannot be resolved\", query_id=20261024_000000_00000_abcde)"+ for identifier in [+ "error 403: b'This tenant is not admitted to the pool yet",+ "name=CLUSTER_OUT_OF_MEMORY,",+ "name=TOO_MANY_REQUESTS_FAILED,",+ "name=PAGE_TRANSPORT_TIMEOUT,",+ ]+ ),],)async def test_suspends_when_a_customer_identifier_spells_an_abort_marker(- self, ateam, anode, asaved_query, adag, identifier+ self, ateam, anode, asaved_query, adag, error):from posthog.temporal.data_modeling.activities.utils import (CONSECUTIVE_FAILURES_TO_SUSPEND,@@ -946,7 +962,6 @@ class TestNodeSuspension:maybe_suspend_node_for_engine,)- error = f"Code: 47. DB::Exception: Missing columns: '{identifier}' while processing query"jobs = [await _make_job(ateam, asaved_query, DataModelingJob.Status.FAILED, error=error)for _ in range(CONSECUTIVE_FAILURES_TO_SUSPEND)diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.pyindex c679ac4e6827..bedeeb31e7f3 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py@@ -412,10 +412,14 @@ class TestStagedAudit:class TestFinalizeOrphanedManagedWarehouseJob:- async def test_marks_orphaned_job_failed_without_touching_node(self):+ @pytest.mark.parametrize(+ "error,cancelled",+ [(RuntimeError("activity died"), False), (CancelledError("activity died"), True)],+ )+ async def test_marks_orphaned_job_failed_without_touching_node(self, error, cancelled):workflow = MaterializeViewWorkflow()with patch.object(temporalio.workflow, "execute_activity", new=AsyncMock()) as execute_activity:- await workflow._finalize_orphaned_managed_warehouse_job("job-123", _inputs(), "activity died")+ await workflow._finalize_orphaned_managed_warehouse_job("job-123", _inputs(), error)execute_activity.assert_awaited_once()assert execute_activity.await_args is not None@@ -426,11 +430,13 @@ class TestFinalizeOrphanedManagedWarehouseJob:# shadow job has no node properties to update — only finalize the job rowassert payload.update_node is Falseassert "activity died" in payload.error+ # a cancelled shadow never got to fail on its own merits, so it must not count toward suspension+ assert payload.cancelled is cancelledasync def test_noop_when_no_managed_warehouse_job(self):workflow = MaterializeViewWorkflow()with patch.object(temporalio.workflow, "execute_activity", new=AsyncMock()) as execute_activity:- await workflow._finalize_orphaned_managed_warehouse_job(None, _inputs(), "activity died")+ await workflow._finalize_orphaned_managed_warehouse_job(None, _inputs(), RuntimeError("activity died"))execute_activity.assert_not_awaited()async def test_finalization_is_best_effort(self):@@ -440,7 +446,7 @@ class TestFinalizeOrphanedManagedWarehouseJob:patch.object(temporalio.workflow, "logger"),):# a failure to finalize must never propagate out of the shadow path- await workflow._finalize_orphaned_managed_warehouse_job("job-123", _inputs(), "activity died")+ await workflow._finalize_orphaned_managed_warehouse_job("job-123", _inputs(), RuntimeError("activity died"))class TestMaybeStageAccountProperties:diff --git a/products/managed_warehouse/backend/tests/test_trino_execution.py b/products/managed_warehouse/backend/tests/test_trino_execution.pyindex aa061a1e3e9f..b15d93fe5b34 100644--- a/products/managed_warehouse/backend/tests/test_trino_execution.py+++ b/products/managed_warehouse/backend/tests/test_trino_execution.py@@ -4,7 +4,7 @@ import threadingimport pytestfrom unittest.mock import MagicMock, patch-from trino.exceptions import TrinoQueryError+from trino.exceptions import HttpError, TrinoQueryError, TrinoUserErrorfrom products.managed_warehouse.backend import trino_execution as execution@@ -77,10 +77,26 @@ async def test_reports_cpu_time() -> None:@pytest.mark.asyncio@pytest.mark.parametrize(- "error_name,expected_attempts",- [("CLUSTER_OUT_OF_MEMORY", 2), ("TOO_MANY_REQUESTS_FAILED", 2), ("EXCEEDED_LOCAL_MEMORY_LIMIT", 1)],+ "error,expected_attempts",+ [+ (TrinoQueryError({"errorName": "CLUSTER_OUT_OF_MEMORY", "message": "Query killed"}), 2),+ (TrinoQueryError({"errorName": "TOO_MANY_REQUESTS_FAILED", "message": "Query killed"}), 2),+ (TrinoQueryError({"errorName": "PAGE_TRANSPORT_TIMEOUT", "message": "Query killed"}), 2),+ (HttpError("error 403: b'This tenant is not admitted to the pool yet; the request was not dispatched'"), 2),+ (HttpError("error 403: b'Forbidden'"), 1),+ (+ TrinoUserError(+ {+ "errorName": "COLUMN_NOT_FOUND",+ "message": "line 1:8: Column 'This tenant is not admitted to the pool yet' cannot be resolved",+ }+ ),+ 1,+ ),+ (TrinoQueryError({"errorName": "EXCEEDED_LOCAL_MEMORY_LIMIT", "message": "Query killed"}), 1),+ ],)-async def test_retries_only_transient_trino_errors(error_name: str, expected_attempts: int) -> None:+async def test_retries_only_transient_trino_errors(error: Exception, expected_attempts: int) -> None:attempts = 0processed: list[int] = []@@ -89,11 +105,11 @@ async def test_retries_only_transient_trino_errors(error_name: str, expected_attattempts += 1control.record_processed_bytes(10)if attempts == 1:- raise TrinoQueryError({"errorName": error_name, "message": "Query killed"})+ raise errorreturn 42if expected_attempts == 1:- with pytest.raises(TrinoQueryError):+ with pytest.raises(type(error)):await execution.run_trino_model_with_retries(execute, query_seconds=60, retry_delays=(0,), on_processed_bytes=processed.append)@@ -106,3 +122,19 @@ async def test_retries_only_transient_trino_errors(error_name: str, expected_att)assert attempts == expected_attemptsassert processed == [10] * expected_attempts+++async def test_deadline_after_a_retry_reports_the_whole_budget() -> None:+ attempts = 0++ def execute(control: execution.TrinoQueryControl) -> int:+ nonlocal attempts+ attempts += 1+ if attempts == 1:+ raise TrinoQueryError({"errorName": "CLUSTER_OUT_OF_MEMORY", "message": "Query killed"})+ assert control.cancelled.wait(10)+ return 42++ with pytest.raises(TimeoutError, match=r"exceeded its 0\.01-minute deadline after 2 attempts"):+ await execution.run_trino_model_with_retries(execute, query_seconds=0.6, retry_delays=(0,))diff --git a/products/managed_warehouse/backend/trino_execution.py b/products/managed_warehouse/backend/trino_execution.pyindex 3ef43be75790..32f0f61b69cc 100644--- a/products/managed_warehouse/backend/trino_execution.py+++ b/products/managed_warehouse/backend/trino_execution.py@@ -7,6 +7,7 @@ from time import monotonicfrom typing import TYPE_CHECKING, TypeVarimport structlog+from trino.exceptions import HttpErrorfrom posthog.sync import database_sync_to_async_pool@@ -21,8 +22,12 @@ TRINO_QUERY_SECONDS = 6 * 60 * 60# Trino stops these queries for a cluster fault, not for the query itself, and asks the client to try# again in a few minutes. The low-memory killer raises CLUSTER_OUT_OF_MEMORY when the whole cluster runs# out of memory. TOO_MANY_REQUESTS_FAILED means a worker node crashed or stopped answering. A query over-# its own memory limit fails with another name.-TRANSIENT_TRINO_ERROR_NAMES = frozenset({"CLUSTER_OUT_OF_MEMORY", "TOO_MANY_REQUESTS_FAILED"})+# its own memory limit fails with another name. PAGE_TRANSPORT_TIMEOUT means workers lost contact while+# passing data between stages.+TRANSIENT_TRINO_ERROR_NAMES = frozenset({"CLUSTER_OUT_OF_MEMORY", "TOO_MANY_REQUESTS_FAILED", "PAGE_TRANSPORT_TIMEOUT"})+# The gateway refuses the request with an HTTP 403 before it reaches a cluster, so a retry cannot repeat any work.+# This is the start of the error text the client builds from the gateway's response body.+TRINO_POOL_NOT_ADMITTED_ERROR_PREFIX = "error 403: b'This tenant is not admitted to the pool yet"TRANSIENT_TRINO_ERROR_RETRY_DELAYS_SECONDS = (2 * 60, 5 * 60)_executor = ThreadPoolExecutor(thread_name_prefix="trino-model")@@ -80,6 +85,8 @@ async def run_trino_model(query_seconds: float = TRINO_QUERY_SECONDS,on_processed_bytes: Callable[[int], None] | None = None,on_cpu_milliseconds: Callable[[int], None] | None = None,+ budget_seconds: float | None = None,+ attempt: int = 1,) -> _Result:control = TrinoQueryControl(query_seconds, on_processed_bytes, on_cpu_milliseconds)task = asyncio.create_task(database_sync_to_async_pool(execute, executor=_executor)(control))@@ -97,14 +104,20 @@ async def run_trino_model(except Exception:breakif isinstance(error, TimeoutError) and timeout.expired():- raise TimeoutError(f"Trino model execution exceeded its {query_seconds / 60:g}-minute deadline") from error+ budget = query_seconds if budget_seconds is None else budget_seconds+ attempts = f" after {attempt} attempts" if attempt > 1 else ""+ raise TimeoutError(+ f"Trino model execution exceeded its {budget / 60:g}-minute deadline{attempts}"+ ) from errorraisefinally:control.finished.set()def is_transient_trino_error(error: BaseException) -> bool:- return getattr(error, "error_name", None) in TRANSIENT_TRINO_ERROR_NAMES+ if getattr(error, "error_name", None) in TRANSIENT_TRINO_ERROR_NAMES:+ return True+ return isinstance(error, HttpError) and str(error).startswith(TRINO_POOL_NOT_ADMITTED_ERROR_PREFIX)async def run_trino_model_with_retries(@@ -120,13 +133,15 @@ async def run_trino_model_with_retries(The usage callbacks receive each statement's final counters, in every attempt."""deadline = monotonic() + query_seconds- for delay in retry_delays:+ for attempt, delay in enumerate(retry_delays, start=1):try:return await run_trino_model(execute,query_seconds=deadline - monotonic(),on_processed_bytes=on_processed_bytes,on_cpu_milliseconds=on_cpu_milliseconds,+ budget_seconds=query_seconds,+ attempt=attempt,)except Exception as error:if not is_transient_trino_error(error) or monotonic() + delay >= deadline:@@ -142,4 +157,6 @@ async def run_trino_model_with_retries(query_seconds=deadline - monotonic(),on_processed_bytes=on_processed_bytes,on_cpu_milliseconds=on_cpu_milliseconds,+ budget_seconds=query_seconds,+ attempt=len(retry_delays) + 1,)