:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): notify when a materialized view is suspended (#122016)
Jovan Sakovic19e59f79db88
diff --git a/posthog/tasks/email.py b/posthog/tasks/email.pyindex aa386eb239ec..cf25670f49e7 100644--- a/posthog/tasks/email.py+++ b/posthog/tasks/email.py@@ -1602,11 +1602,13 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],@shared_task(**EMAIL_TASK_KWARGS)@skip_team_scope_audit-def send_matview_failure_immediate_email(team_id: int, saved_query_id: str, job_id: str) -> None:+def send_matview_failure_immediate_email(+ team_id: int, saved_query_id: str, job_id: str, suspended: bool = False+) -> None:"""Email members who asked for a materialization failure email as it happens.- Dispatched on the first failure of a streak only; the job-scoped campaign key- makes redelivery idempotent per recipient.+ Dispatched on the first failure of a streak, and again when the streak suspends the view;+ the job-scoped campaign key makes redelivery idempotent per recipient."""if not is_email_available(with_absolute_urls=True):@@ -1639,15 +1641,22 @@ def send_matview_failure_immediate_email(team_id: int, saved_query_id: str, job_if len(error) > MAX_ERROR_CHARS:error = error[: MAX_ERROR_CHARS - 3] + "..."+ if suspended:+ campaign_key = f"matview_suspended_{saved_query_id}_{job_id}"+ subject = f"PostHog: Materialized view '{saved_query.name}' was suspended in {team.name}"+ else:+ campaign_key = f"matview_failure_immediate_{saved_query_id}_{job_id}"+ subject = f"PostHog: Materialized view '{saved_query.name}' failed in {team.name}"message = EmailMessage(- campaign_key=f"matview_failure_immediate_{saved_query_id}_{job_id}",- subject=f"PostHog: Materialized view '{saved_query.name}' failed in {team.name}",+ campaign_key=campaign_key,+ subject=subject,template_name="saved_query_materialization_failure",template_context={"team": team,"saved_query_name": saved_query.name,"saved_query_id": str(saved_query.id),"error": error,+ "suspended": suspended,},)for membership in memberships_to_email:diff --git a/posthog/tasks/test/test_email.py b/posthog/tasks/test/test_email.pyindex 1afb6815a442..bdb877ec111a 100644--- a/posthog/tasks/test/test_email.py+++ b/posthog/tasks/test/test_email.py@@ -2819,13 +2819,27 @@ class TestEmail(APIBaseTest, ClickhouseTestMixin):"Code: 241. DB::Exception: Memory limit (for query) exceeded",None,"Code: 241. DB::Exception: Memory limit (for query) exceeded",+ False,),- ("long_job_error_is_truncated", "B" * 500, None, "B" * 252 + "..."),- ("falls_back_to_the_saved_query_error", "", "Query exceeded timeout limit", "Query exceeded timeout limit"),+ ("long_job_error_is_truncated", "B" * 500, None, "B" * 252 + "...", False),+ (+ "falls_back_to_the_saved_query_error",+ "",+ "Query exceeded timeout limit",+ "Query exceeded timeout limit",+ False,+ ),+ ("suspension_says_so_and_how_to_resume", "boom", None, "boom", True),])def test_send_matview_failure_immediate_email_includes_the_error(- self, MockEmailMessage: MagicMock, _name: str, job_error: str, latest_error: str | None, expected_error: str+ self,+ MockEmailMessage: MagicMock,+ _name: str,+ job_error: str,+ latest_error: str | None,+ expected_error: str,+ suspended: bool,) -> None:mocked_email_messages = mock_email_messages(MockEmailMessage)@@ -2850,11 +2864,16 @@ class TestEmail(APIBaseTest, ClickhouseTestMixin):last_run_at=timezone.now(),)- send_matview_failure_immediate_email(self.team.id, str(sq.id), str(job.id))+ send_matview_failure_immediate_email(self.team.id, str(sq.id), str(job.id), suspended=suspended)assert len(mocked_email_messages) == 1- assert mocked_email_messages[0].properties["error"] == expected_error- assert expected_error[:80] in mocked_email_messages[0].html_body+ message = mocked_email_messages[0]+ assert message.properties["error"] == expected_error+ assert expected_error[:80] in message.html_body+ # A distinct key, or the suspension email dedupes against a failure email for the same job.+ assert message.campaign_key.startswith("matview_suspended_" if suspended else "matview_failure_immediate_")+ assert ("was suspended" in message.subject) == suspended+ assert ("select Resume" in message.html_body) == suspendeddef test_send_matview_failure_digest_splits_by_view_access(self, MockEmailMessage: MagicMock) -> None:mocked_email_messages = mock_email_messages(MockEmailMessage)diff --git a/posthog/templates/email/saved_query_materialization_failure.html b/posthog/templates/email/saved_query_materialization_failure.htmlindex f4f6da6c2e80..d40969059135 100644--- a/posthog/templates/email/saved_query_materialization_failure.html+++ b/posthog/templates/email/saved_query_materialization_failure.html@@ -1,9 +1,27 @@{% extends "email/base.html" %}-{% block preheader %}Your materialized view failed to sync{% endblock %}-{% block heading %}Materialized view '{{ saved_query_name }}' sync failed{% endblock %}+{% block preheader %}+ {% if suspended %}+ Your materialized view stopped running after repeated failures+ {% else %}+ Your materialized view failed to sync+ {% endif %}+{% endblock %}+{% block heading %}+ {% if suspended %}+ Materialized view '{{ saved_query_name }}' was suspended+ {% else %}+ Materialized view '{{ saved_query_name }}' sync failed+ {% endif %}+{% endblock %}{% block section %}<p>- Your materialized view <b>{{ saved_query_name }}</b> failed to sync.+ {% if suspended %}+ Your materialized view <b>{{ saved_query_name }}</b> failed several runs in a row, so we stopped running it.+ Its data stays as it was after the last successful run.+ Fix the query, then open the view and select Resume to start it again.+ {% else %}+ Your materialized view <b>{{ saved_query_name }}</b> failed to sync.+ {% endif %}{% if error %}The last run reported this error:{% else %}diff --git a/posthog/temporal/data_modeling/activities/fail_materialization.py b/posthog/temporal/data_modeling/activities/fail_materialization.pyindex 75f271f23537..d036f7514baa 100644--- a/posthog/temporal/data_modeling/activities/fail_materialization.py+++ b/posthog/temporal/data_modeling/activities/fail_materialization.py@@ -19,13 +19,14 @@ from products.data_modeling.backend.facade.models import ()from ..metrics import get_node_suspended_metric-from .notify_materialization_failure import maybe_notify_materialization_failure+from .notify_materialization_failure import maybe_notify_materialization_failure, notify_node_suspendedfrom .utils import (CONSECUTIVE_FAILURES_TO_SUSPEND,bind_data_modeling_log_context,maybe_suspend_node_for_engine,strip_hostname_from_error,update_node_system_properties,+ was_suspended_by_job,)LOGGER = get_logger(__name__)@@ -125,6 +126,7 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> streturn job.run_modeerror = inputs.errorsaved_query = None+ suspended = Falsetry:saved_query = await _get_saved_query_for_job(job)if saved_query is None:@@ -136,7 +138,7 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> st)await _revert_materialization_on_unknown_table(job, saved_query)else:- suspended = await maybe_suspend_node_for_engine(+ if await maybe_suspend_node_for_engine(node_id=inputs.node_id,team_id=inputs.team_id,dag_id=inputs.dag_id,@@ -144,12 +146,21 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> stengine=DataModelingJobEngine.CLICKHOUSE,reason=strip_hostname_from_error(error),job_id=inputs.job_id,- )- if suspended:+ ):+ suspended = Trueget_node_suspended_metric(DataModelingJobEngine.CLICKHOUSE.value).add(1)await logger.ainfo(f"Suspended node {inputs.node_id} (clickhouse) after {CONSECUTIVE_FAILURES_TO_SUSPEND} consecutive failures",)+ else:+ # A retry after the marker was saved but before anyone was told still owes the notice.+ suspended = await was_suspended_by_job(+ node_id=inputs.node_id,+ team_id=inputs.team_id,+ dag_id=inputs.dag_id,+ engine=DataModelingJobEngine.CLICKHOUSE,+ job_id=inputs.job_id,+ )except Exception as e:capture_exception(e)@@ -161,7 +172,12 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> st# needs telling, so it must not take the notification down with it.if saved_query is not None and not inputs.cancelled:try:- notified = await maybe_notify_materialization_failure(job, saved_query, inputs.team_id)+ if suspended:+ # Replaces the failure notification for this run, so nobody gets two for one event.+ await notify_node_suspended(job, saved_query, inputs.team_id, strip_hostname_from_error(error))+ notified = True+ else:+ notified = await maybe_notify_materialization_failure(job, saved_query, inputs.team_id)if notified:await logger.ainfo(f"Sent materialization failure notification for node {inputs.node_id}")except Exception as e:diff --git a/posthog/temporal/data_modeling/activities/notify_materialization_failure.py b/posthog/temporal/data_modeling/activities/notify_materialization_failure.pyindex 4096f98a9fbf..1411a7fce71d 100644--- a/posthog/temporal/data_modeling/activities/notify_materialization_failure.py+++ b/posthog/temporal/data_modeling/activities/notify_materialization_failure.py@@ -170,6 +170,37 @@ def maybe_notify_materialization_failure(return True+@database_sync_to_async_pool+def notify_node_suspended(+ job: DataModelingJob, saved_query: DataWarehouseSavedQuery, team_id: int, reason: str+) -> None:+ """Tell the team a view stopped running, which is the moment that needs someone to act.++ The failure notifications go quiet after a streak's first failure, so without this the+ view would stop updating with nothing said.+ """+ send_matview_failure_immediate_email.delay(team_id, str(saved_query.id), str(job.id), suspended=True)+ create_notification(+ NotificationData(+ team_id=team_id,+ notification_type=NotificationType.MATERIALIZATION_FAILURE,+ priority=Priority.NORMAL,+ title=f"{saved_query.name} was suspended after repeated failures"[:255],+ body=f"Fix the query, then open the view and select Resume. Last error: {reason}"[:400],+ target_type=TargetType.TEAM,+ target_id=str(team_id),+ resource_type="warehouse_objects",+ resource_id=str(saved_query.id),+ source_url=f"/project/{team_id}/sql?open_view={saved_query.id}",+ source_id=str(job.id),+ # A retry of the suspending run sends again, so this dedupes it. Sharing the failure+ # notification's key also means one run never produces both.+ idempotency_key=_dedupe_key([_FailedView(job=job, saved_query=saved_query)]),+ resolver=_SavedQueryViewers(saved_query),+ )+ )++def _failure_copy(views: list[_FailedView]) -> tuple[str, str]:names = [view.saved_query.name for view in views]if len(names) == 1:diff --git a/posthog/temporal/data_modeling/activities/utils.py b/posthog/temporal/data_modeling/activities/utils.pyindex db747951918b..fbf6c0aeac98 100644--- a/posthog/temporal/data_modeling/activities/utils.py+++ b/posthog/temporal/data_modeling/activities/utils.py@@ -18,6 +18,7 @@ from products.data_modeling.backend.facade.api import (mark_node_suspended,query_fingerprint,suspension_reset_at,+ suspension_state,)from products.data_modeling.backend.facade.models import (DataModelingJob,@@ -245,6 +246,16 @@ def maybe_suspend_node_for_engine(return True+@database_sync_to_async_pool+def was_suspended_by_job(*, node_id: str, team_id: int, dag_id: str, engine: str, job_id: str) -> bool:+ """True when this job's failure is the one recorded as suspending the node.++ A retry of the failing activity no longer sees the transition, so this is how it still knows.+ """+ node = Node.objects.get(id=node_id, team_id=team_id, dag_id=dag_id)+ return (suspension_state(node).get(str(engine)) or {}).get("job_id") == job_id++@database_sync_to_async_pooldef clear_node_suspension_for_engine(*, node_id: str, team_id: int, dag_id: str, engine: str) -> bool:with transaction.atomic():diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.pyindex 9342285717dd..4dbfa7fd4e37 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py@@ -62,6 +62,7 @@ from products.customer_analytics.backend.facade.temporal import stage_warehouse_from products.customer_analytics.backend.facade.temporal_contracts import StageAccountPropertySyncInputfrom products.data_modeling.backend.facade.api import (TRINO_INCREMENTAL_SCOPE,+ clear_node_suspension,compute_enrichment_hash,definition_fingerprint,get_incremental_config,@@ -419,10 +420,50 @@ class TestFailMaterializationActivity:job_id=str(current_job.id),error=error,)- await activity_environment.run(fail_materialization_activity, inputs)+ with (+ unittest.mock.patch(+ "posthog.temporal.data_modeling.activities.notify_materialization_failure.create_notification"+ ) as mock_create,+ unittest.mock.patch(+ "posthog.temporal.data_modeling.activities.notify_materialization_failure.send_matview_failure_immediate_email"+ ) as mock_email,+ ):+ # The first attempt saves the marker, but the in-app notice never lands.+ mock_create.side_effect = RuntimeError("notifications unavailable")+ await activity_environment.run(fail_materialization_activity, inputs)++ await database_sync_to_async(anode.refresh_from_db)()+ assert is_node_suspended(anode, DataModelingJobEngine.CLICKHOUSE) is True+ suspended_email = unittest.mock.call(ateam.pk, str(asaved_query.id), str(current_job.id), suspended=True)+ assert mock_email.delay.call_args_list == [suspended_email]++ # The retry no longer sees the transition but still owes the notice. It resends with the+ # same job, so the email's campaign key and the in-app idempotency key both dedupe.+ mock_create.side_effect = None+ await activity_environment.run(fail_materialization_activity, inputs)+ assert mock_email.delay.call_args_list == [suspended_email, suspended_email]+ data = mock_create.call_args.args[0]+ assert "suspended" in data.title+ assert "Last error: " + error in data.body+ assert data.target_type == TargetType.TEAM+ assert {call.args[0].idempotency_key for call in mock_create.call_args_list} == {data.idempotency_key}++ # Resuming and failing the full streak again is a new suspension, so it notifies again.+ await database_sync_to_async(clear_node_suspension)(+ anode, engine=DataModelingJobEngine.CLICKHOUSE, by="api"+ )+ await database_sync_to_async(anode.save)()+ for _ in range(4):+ await _make_job(ateam, asaved_query, DataModelingJob.Status.FAILED, error=previous_error)+ next_job = await _make_job(ateam, asaved_query, DataModelingJob.Status.RUNNING)+ await activity_environment.run(fail_materialization_activity, replace(inputs, job_id=str(next_job.id)))await database_sync_to_async(anode.refresh_from_db)()assert is_node_suspended(anode, DataModelingJobEngine.CLICKHOUSE) is True+ assert mock_email.delay.call_args_list[-1] == unittest.mock.call(+ ateam.pk, str(asaved_query.id), str(next_job.id), suspended=True+ )+ assert mock_email.delay.call_count == 3@pytest.mark.parametrize("previous_status,parent_workflow_id,with_runner,expect_email,expect_in_app",