Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): notify when a materialized view is suspended (#122016)

Jovan Sakovic19e59f79db88
diff --git a/posthog/tasks/email.py b/posthog/tasks/email.py
index aa386eb239ec..cf25670f49e7 100644
--- a/posthog/tasks/email.py
+++ b/posthog/tasks/email.py
@@ -1602,11 +1602,13 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],
@shared_task(**EMAIL_TASK_KWARGS)
@skip_team_scope_audit
-def send_matview_failure_immediate_email(team_id: int, saved_query_id: str, job_id: str) -> None:
+def send_matview_failure_immediate_email(
+ team_id: int, saved_query_id: str, job_id: str, suspended: bool = False
+) -> None:
"""Email members who asked for a materialization failure email as it happens.
- Dispatched on the first failure of a streak only; the job-scoped campaign key
- makes redelivery idempotent per recipient.
+ Dispatched on the first failure of a streak, and again when the streak suspends the view;
+ the job-scoped campaign key makes redelivery idempotent per recipient.
"""
if not is_email_available(with_absolute_urls=True):
@@ -1639,15 +1641,22 @@ def send_matview_failure_immediate_email(team_id: int, saved_query_id: str, job_
if len(error) > MAX_ERROR_CHARS:
error = error[: MAX_ERROR_CHARS - 3] + "..."
+ if suspended:
+ campaign_key = f"matview_suspended_{saved_query_id}_{job_id}"
+ subject = f"PostHog: Materialized view '{saved_query.name}' was suspended in {team.name}"
+ else:
+ campaign_key = f"matview_failure_immediate_{saved_query_id}_{job_id}"
+ subject = f"PostHog: Materialized view '{saved_query.name}' failed in {team.name}"
message = EmailMessage(
- campaign_key=f"matview_failure_immediate_{saved_query_id}_{job_id}",
- subject=f"PostHog: Materialized view '{saved_query.name}' failed in {team.name}",
+ campaign_key=campaign_key,
+ subject=subject,
template_name="saved_query_materialization_failure",
template_context={
"team": team,
"saved_query_name": saved_query.name,
"saved_query_id": str(saved_query.id),
"error": error,
+ "suspended": suspended,
},
)
for membership in memberships_to_email:
diff --git a/posthog/tasks/test/test_email.py b/posthog/tasks/test/test_email.py
index 1afb6815a442..bdb877ec111a 100644
--- a/posthog/tasks/test/test_email.py
+++ b/posthog/tasks/test/test_email.py
@@ -2819,13 +2819,27 @@ class TestEmail(APIBaseTest, ClickhouseTestMixin):
"Code: 241. DB::Exception: Memory limit (for query) exceeded",
None,
"Code: 241. DB::Exception: Memory limit (for query) exceeded",
+ False,
),
- ("long_job_error_is_truncated", "B" * 500, None, "B" * 252 + "..."),
- ("falls_back_to_the_saved_query_error", "", "Query exceeded timeout limit", "Query exceeded timeout limit"),
+ ("long_job_error_is_truncated", "B" * 500, None, "B" * 252 + "...", False),
+ (
+ "falls_back_to_the_saved_query_error",
+ "",
+ "Query exceeded timeout limit",
+ "Query exceeded timeout limit",
+ False,
+ ),
+ ("suspension_says_so_and_how_to_resume", "boom", None, "boom", True),
]
)
def test_send_matview_failure_immediate_email_includes_the_error(
- self, MockEmailMessage: MagicMock, _name: str, job_error: str, latest_error: str | None, expected_error: str
+ self,
+ MockEmailMessage: MagicMock,
+ _name: str,
+ job_error: str,
+ latest_error: str | None,
+ expected_error: str,
+ suspended: bool,
) -> None:
mocked_email_messages = mock_email_messages(MockEmailMessage)
@@ -2850,11 +2864,16 @@ class TestEmail(APIBaseTest, ClickhouseTestMixin):
last_run_at=timezone.now(),
)
- send_matview_failure_immediate_email(self.team.id, str(sq.id), str(job.id))
+ send_matview_failure_immediate_email(self.team.id, str(sq.id), str(job.id), suspended=suspended)
assert len(mocked_email_messages) == 1
- assert mocked_email_messages[0].properties["error"] == expected_error
- assert expected_error[:80] in mocked_email_messages[0].html_body
+ message = mocked_email_messages[0]
+ assert message.properties["error"] == expected_error
+ assert expected_error[:80] in message.html_body
+ # A distinct key, or the suspension email dedupes against a failure email for the same job.
+ assert message.campaign_key.startswith("matview_suspended_" if suspended else "matview_failure_immediate_")
+ assert ("was suspended" in message.subject) == suspended
+ assert ("select Resume" in message.html_body) == suspended
def test_send_matview_failure_digest_splits_by_view_access(self, MockEmailMessage: MagicMock) -> None:
mocked_email_messages = mock_email_messages(MockEmailMessage)
diff --git a/posthog/templates/email/saved_query_materialization_failure.html b/posthog/templates/email/saved_query_materialization_failure.html
index f4f6da6c2e80..d40969059135 100644
--- a/posthog/templates/email/saved_query_materialization_failure.html
+++ b/posthog/templates/email/saved_query_materialization_failure.html
@@ -1,9 +1,27 @@
{% extends "email/base.html" %}
-{% block preheader %}Your materialized view failed to sync{% endblock %}
-{% block heading %}Materialized view '{{ saved_query_name }}' sync failed{% endblock %}
+{% block preheader %}
+ {% if suspended %}
+ Your materialized view stopped running after repeated failures
+ {% else %}
+ Your materialized view failed to sync
+ {% endif %}
+{% endblock %}
+{% block heading %}
+ {% if suspended %}
+ Materialized view '{{ saved_query_name }}' was suspended
+ {% else %}
+ Materialized view '{{ saved_query_name }}' sync failed
+ {% endif %}
+{% endblock %}
{% block section %}
<p>
- Your materialized view <b>{{ saved_query_name }}</b> failed to sync.
+ {% if suspended %}
+ Your materialized view <b>{{ saved_query_name }}</b> failed several runs in a row, so we stopped running it.
+ Its data stays as it was after the last successful run.
+ Fix the query, then open the view and select Resume to start it again.
+ {% else %}
+ Your materialized view <b>{{ saved_query_name }}</b> failed to sync.
+ {% endif %}
{% if error %}
The last run reported this error:
{% else %}
diff --git a/posthog/temporal/data_modeling/activities/fail_materialization.py b/posthog/temporal/data_modeling/activities/fail_materialization.py
index 75f271f23537..d036f7514baa 100644
--- a/posthog/temporal/data_modeling/activities/fail_materialization.py
+++ b/posthog/temporal/data_modeling/activities/fail_materialization.py
@@ -19,13 +19,14 @@ from products.data_modeling.backend.facade.models import (
)
from ..metrics import get_node_suspended_metric
-from .notify_materialization_failure import maybe_notify_materialization_failure
+from .notify_materialization_failure import maybe_notify_materialization_failure, notify_node_suspended
from .utils import (
CONSECUTIVE_FAILURES_TO_SUSPEND,
bind_data_modeling_log_context,
maybe_suspend_node_for_engine,
strip_hostname_from_error,
update_node_system_properties,
+ was_suspended_by_job,
)
LOGGER = get_logger(__name__)
@@ -125,6 +126,7 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> st
return job.run_mode
error = inputs.error
saved_query = None
+ suspended = False
try:
saved_query = await _get_saved_query_for_job(job)
if saved_query is None:
@@ -136,7 +138,7 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> st
)
await _revert_materialization_on_unknown_table(job, saved_query)
else:
- suspended = await maybe_suspend_node_for_engine(
+ if await maybe_suspend_node_for_engine(
node_id=inputs.node_id,
team_id=inputs.team_id,
dag_id=inputs.dag_id,
@@ -144,12 +146,21 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> st
engine=DataModelingJobEngine.CLICKHOUSE,
reason=strip_hostname_from_error(error),
job_id=inputs.job_id,
- )
- if suspended:
+ ):
+ suspended = True
get_node_suspended_metric(DataModelingJobEngine.CLICKHOUSE.value).add(1)
await logger.ainfo(
f"Suspended node {inputs.node_id} (clickhouse) after {CONSECUTIVE_FAILURES_TO_SUSPEND} consecutive failures",
)
+ else:
+ # A retry after the marker was saved but before anyone was told still owes the notice.
+ suspended = await was_suspended_by_job(
+ node_id=inputs.node_id,
+ team_id=inputs.team_id,
+ dag_id=inputs.dag_id,
+ engine=DataModelingJobEngine.CLICKHOUSE,
+ job_id=inputs.job_id,
+ )
except Exception as e:
capture_exception(e)
@@ -161,7 +172,12 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> st
# needs telling, so it must not take the notification down with it.
if saved_query is not None and not inputs.cancelled:
try:
- notified = await maybe_notify_materialization_failure(job, saved_query, inputs.team_id)
+ if suspended:
+ # Replaces the failure notification for this run, so nobody gets two for one event.
+ await notify_node_suspended(job, saved_query, inputs.team_id, strip_hostname_from_error(error))
+ notified = True
+ else:
+ notified = await maybe_notify_materialization_failure(job, saved_query, inputs.team_id)
if notified:
await logger.ainfo(f"Sent materialization failure notification for node {inputs.node_id}")
except Exception as e:
diff --git a/posthog/temporal/data_modeling/activities/notify_materialization_failure.py b/posthog/temporal/data_modeling/activities/notify_materialization_failure.py
index 4096f98a9fbf..1411a7fce71d 100644
--- a/posthog/temporal/data_modeling/activities/notify_materialization_failure.py
+++ b/posthog/temporal/data_modeling/activities/notify_materialization_failure.py
@@ -170,6 +170,37 @@ def maybe_notify_materialization_failure(
return True
+@database_sync_to_async_pool
+def notify_node_suspended(
+ job: DataModelingJob, saved_query: DataWarehouseSavedQuery, team_id: int, reason: str
+) -> None:
+ """Tell the team a view stopped running, which is the moment that needs someone to act.
+
+ The failure notifications go quiet after a streak's first failure, so without this the
+ view would stop updating with nothing said.
+ """
+ send_matview_failure_immediate_email.delay(team_id, str(saved_query.id), str(job.id), suspended=True)
+ create_notification(
+ NotificationData(
+ team_id=team_id,
+ notification_type=NotificationType.MATERIALIZATION_FAILURE,
+ priority=Priority.NORMAL,
+ title=f"{saved_query.name} was suspended after repeated failures"[:255],
+ body=f"Fix the query, then open the view and select Resume. Last error: {reason}"[:400],
+ target_type=TargetType.TEAM,
+ target_id=str(team_id),
+ resource_type="warehouse_objects",
+ resource_id=str(saved_query.id),
+ source_url=f"/project/{team_id}/sql?open_view={saved_query.id}",
+ source_id=str(job.id),
+ # A retry of the suspending run sends again, so this dedupes it. Sharing the failure
+ # notification's key also means one run never produces both.
+ idempotency_key=_dedupe_key([_FailedView(job=job, saved_query=saved_query)]),
+ resolver=_SavedQueryViewers(saved_query),
+ )
+ )
+
+
def _failure_copy(views: list[_FailedView]) -> tuple[str, str]:
names = [view.saved_query.name for view in views]
if len(names) == 1:
diff --git a/posthog/temporal/data_modeling/activities/utils.py b/posthog/temporal/data_modeling/activities/utils.py
index db747951918b..fbf6c0aeac98 100644
--- a/posthog/temporal/data_modeling/activities/utils.py
+++ b/posthog/temporal/data_modeling/activities/utils.py
@@ -18,6 +18,7 @@ from products.data_modeling.backend.facade.api import (
mark_node_suspended,
query_fingerprint,
suspension_reset_at,
+ suspension_state,
)
from products.data_modeling.backend.facade.models import (
DataModelingJob,
@@ -245,6 +246,16 @@ def maybe_suspend_node_for_engine(
return True
+@database_sync_to_async_pool
+def was_suspended_by_job(*, node_id: str, team_id: int, dag_id: str, engine: str, job_id: str) -> bool:
+ """True when this job's failure is the one recorded as suspending the node.
+
+ A retry of the failing activity no longer sees the transition, so this is how it still knows.
+ """
+ node = Node.objects.get(id=node_id, team_id=team_id, dag_id=dag_id)
+ return (suspension_state(node).get(str(engine)) or {}).get("job_id") == job_id
+
+
@database_sync_to_async_pool
def clear_node_suspension_for_engine(*, node_id: str, team_id: int, dag_id: str, engine: str) -> bool:
with transaction.atomic():
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
index 9342285717dd..4dbfa7fd4e37 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
@@ -62,6 +62,7 @@ from products.customer_analytics.backend.facade.temporal import stage_warehouse_
from products.customer_analytics.backend.facade.temporal_contracts import StageAccountPropertySyncInput
from products.data_modeling.backend.facade.api import (
TRINO_INCREMENTAL_SCOPE,
+ clear_node_suspension,
compute_enrichment_hash,
definition_fingerprint,
get_incremental_config,
@@ -419,10 +420,50 @@ class TestFailMaterializationActivity:
job_id=str(current_job.id),
error=error,
)
- await activity_environment.run(fail_materialization_activity, inputs)
+ with (
+ unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.notify_materialization_failure.create_notification"
+ ) as mock_create,
+ unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.notify_materialization_failure.send_matview_failure_immediate_email"
+ ) as mock_email,
+ ):
+ # The first attempt saves the marker, but the in-app notice never lands.
+ mock_create.side_effect = RuntimeError("notifications unavailable")
+ await activity_environment.run(fail_materialization_activity, inputs)
+
+ await database_sync_to_async(anode.refresh_from_db)()
+ assert is_node_suspended(anode, DataModelingJobEngine.CLICKHOUSE) is True
+ suspended_email = unittest.mock.call(ateam.pk, str(asaved_query.id), str(current_job.id), suspended=True)
+ assert mock_email.delay.call_args_list == [suspended_email]
+
+ # The retry no longer sees the transition but still owes the notice. It resends with the
+ # same job, so the email's campaign key and the in-app idempotency key both dedupe.
+ mock_create.side_effect = None
+ await activity_environment.run(fail_materialization_activity, inputs)
+ assert mock_email.delay.call_args_list == [suspended_email, suspended_email]
+ data = mock_create.call_args.args[0]
+ assert "suspended" in data.title
+ assert "Last error: " + error in data.body
+ assert data.target_type == TargetType.TEAM
+ assert {call.args[0].idempotency_key for call in mock_create.call_args_list} == {data.idempotency_key}
+
+ # Resuming and failing the full streak again is a new suspension, so it notifies again.
+ await database_sync_to_async(clear_node_suspension)(
+ anode, engine=DataModelingJobEngine.CLICKHOUSE, by="api"
+ )
+ await database_sync_to_async(anode.save)()
+ for _ in range(4):
+ await _make_job(ateam, asaved_query, DataModelingJob.Status.FAILED, error=previous_error)
+ next_job = await _make_job(ateam, asaved_query, DataModelingJob.Status.RUNNING)
+ await activity_environment.run(fail_materialization_activity, replace(inputs, job_id=str(next_job.id)))
await database_sync_to_async(anode.refresh_from_db)()
assert is_node_suspended(anode, DataModelingJobEngine.CLICKHOUSE) is True
+ assert mock_email.delay.call_args_list[-1] == unittest.mock.call(
+ ateam.pk, str(asaved_query.id), str(next_job.id), suspended=True
+ )
+ assert mock_email.delay.call_count == 3
@pytest.mark.parametrize(
"previous_status,parent_workflow_id,with_runner,expect_email,expect_in_app",