:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): group failure digest into new, ongoing and action sections (#121724)
Jovan Sakovicc98a6a21cbc3
diff --git a/posthog/tasks/email.py b/posthog/tasks/email.pyindex edb748ced52d..aa386eb239ec 100644--- a/posthog/tasks/email.py+++ b/posthog/tasks/email.py@@ -1,12 +1,14 @@import uuidimport hashlibimport datetime+import collectionsfrom enum import Enumfrom typing import Any, Literal, Optional, castfrom django.conf import settingsfrom django.db import transaction-from django.db.models import OuterRef, Subquery+from django.db.models import Count, OuterRef, Subquery, Value+from django.db.models.functions import Coalescefrom django.utils import timezoneimport structlog@@ -60,6 +62,7 @@ from products.cdp.backend.models.hog_functions.hog_function import HogFunctionfrom products.cdp.backend.models.plugin import Plugin, PluginConfigfrom products.conversations.backend.models import Ticketfrom products.data_modeling.backend.facade.api import (+ declared_targets_by_saved_query,suspended_saved_query_ids_by_team,suspension_state_for_saved_query,)@@ -1333,6 +1336,46 @@ def send_warehouse_destination_paused(MAX_VIEWS_PER_DIGEST_EMAIL = 30MAX_ERROR_CHARS = 255+# Most urgent first: this order drives the email layout, the subject line and which rows the cap keeps.+DIGEST_SECTIONS: dict[str, dict[str, Any]] = {+ "action": {+ "title": "Needs action",+ "description": "These views stopped running. They won't update until you fix them.",+ "color": "#dc3545",+ "verbs": ("needs action", "need action"),+ },+ "new": {+ "title": "Started failing",+ "description": "These started failing in the last 24 hours. They retry on their next scheduled run, so a temporary problem can clear on its own.",+ "color": "#fd7e14",+ "verbs": ("started failing", "started failing"),+ },+ "ongoing": {+ "title": "Still failing",+ "description": "These have failed every run for more than a day. Retrying hasn't fixed them, so they likely need a change.",+ "color": "#9ca3af",+ "verbs": ("is still failing", "are still failing"),+ },+}+DIGEST_SECTION_ORDER = tuple(DIGEST_SECTIONS)+# Older than any job, so a view that never succeeded counts every failure in its streak.+STREAK_FLOOR_BEFORE_ANY_RUN = datetime.datetime(1970, 1, 1, tzinfo=datetime.UTC)+++def _matview_digest_summary(counts: dict[str, int]) -> str:+ """Count each section, e.g. "1 materialized view needs action, 3 started failing"."""+ parts: list[str] = []+ for key in DIGEST_SECTION_ORDER:+ count = counts.get(key, 0)+ if not count:+ continue+ verb = DIGEST_SECTIONS[key]["verbs"][count != 1]+ if not parts:+ noun = "materialized view" if count == 1 else "materialized views"+ parts.append(f"{count} {noun} {verb}")+ else:+ parts.append(f"{count} {verb}")+ return ", ".join(parts) or "Materialized view failures"@shared_task(ignore_result=True)@@ -1419,7 +1462,25 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],returnall_ids = list(set(failed_query_ids + suspended_query_ids))- queries = {str(sq.id): sq for sq in DataWarehouseSavedQuery.objects.filter(id__in=all_ids, team_id=team_id)}+ clickhouse_jobs = DataModelingJob.objects.filter(+ saved_query_id=OuterRef("id"), engine=DataModelingJobEngine.CLICKHOUSE, last_run_at__isnull=False+ )+ last_success_at = (+ clickhouse_jobs.filter(status=DataModelingJob.Status.COMPLETED).order_by("-last_run_at").values("last_run_at")+ )+ streak_jobs = clickhouse_jobs.filter(status=DataModelingJob.Status.FAILED, last_run_at__gt=OuterRef("streak_floor"))+ queries: dict[str, DataWarehouseSavedQuery] = {+ str(sq.id): sq+ for sq in DataWarehouseSavedQuery.objects.filter(id__in=all_ids, team_id=team_id).annotate(+ streak_floor=Coalesce(Subquery(last_success_at[:1]), Value(STREAK_FLOOR_BEFORE_ANY_RUN)),+ failing_since=Subquery(streak_jobs.order_by("last_run_at").values("last_run_at")[:1]),+ failed_runs=Subquery(+ streak_jobs.order_by().values("saved_query_id").annotate(n=Count("id")).values("n")[:1]+ ),+ )+ }+ new_failure_cutoff = timezone.now() - datetime.timedelta(hours=24)+ declared_targets = declared_targets_by_saved_query(team_id, all_ids)latest_jobs: dict[str, DataModelingJob] = {}for latest_job in (@@ -1447,6 +1508,21 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],if len(error) > MAX_ERROR_CHARS:error = error[: MAX_ERROR_CHARS - 3] + "..."run_at = (job.last_run_at if job else None) or sq.last_run_at+ # `is_materialized` stays True after a pause, so read the cadence and the table instead.+ # Pausing clears the cadence and keeps the table; reverting clears both.+ if qid in declared_targets or sq.sync_frequency_interval is not None:+ schedule = "scheduled"+ elif sq.table_id is not None:+ schedule = "paused"+ else:+ schedule = "reverted"+ failing_since: datetime.datetime | None = getattr(sq, "failing_since", None) or run_at+ if suspended or schedule != "scheduled":+ section = "action"+ elif failing_since is None or failing_since >= new_failure_cutoff:+ section = "new"+ else:+ section = "ongoing"views.append({"id": str(sq.id),@@ -1454,11 +1530,11 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],"error": error,"last_run_at": run_at.strftime("%b %d, %H:%M UTC") if run_at else "Unknown","last_run_at_ts": run_at.timestamp() if run_at else 0,+ "failing_since": failing_since.strftime("%b %d, %H:%M UTC") if failing_since else "Unknown",+ "failed_runs": getattr(sq, "failed_runs", None) or 1,+ "section": section,"suspended": suspended,- # Reverting clears the schedule, so promising a retry here would be false. `is not- # False` keeps a never-written flag on the retrying side, as `exclude(deleted=True)`- # does above.- "scheduled": sq.is_materialized is not False,+ "schedule": schedule,"url": f"{settings.SITE_URL}/project/{team_id}/sql?open_view={sq.id}",})@@ -1467,8 +1543,14 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],logger.warning("No failed or suspended views found")return- # Rows needing action outrank rows that heal themselves, so the cap keeps them.- views.sort(key=lambda v: (not v["suspended"], v["scheduled"], -cast(float, v["last_run_at_ts"])))+ # Rows needing action outrank new failures, which outrank old ones, so the cap keeps them.+ views.sort(+ key=lambda v: (+ DIGEST_SECTION_ORDER.index(v["section"]),+ not v["suspended"],+ -cast(float, v["last_run_at_ts"]),+ )+ )for v in views:v.pop("last_run_at_ts", None)@@ -1479,15 +1561,27 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],omitted_count = max(0, len(visible_views) - MAX_VIEWS_PER_DIGEST_EMAIL)listed_views = visible_views[:MAX_VIEWS_PER_DIGEST_EMAIL]+ # Count before the cap, so the subject reflects every failing view the reader can see.+ summary = _matview_digest_summary(collections.Counter(v["section"] for v in visible_views))+ sections = [+ {+ "key": key,+ "title": DIGEST_SECTIONS[key]["title"],+ "description": DIGEST_SECTIONS[key]["description"],+ "color": DIGEST_SECTIONS[key]["color"],+ "views": [v for v in listed_views if v["section"] == key],+ }+ for key in DIGEST_SECTION_ORDER+ ]message = EmailMessage(campaign_key=campaign_key,- subject=f"PostHog: Materialized view failures in {team.name}",+ subject=f"PostHog: {summary} in {team.name}",template_name="matview_failure_digest",template_context={"team": team,"views": listed_views,- "has_suspended": any(v["suspended"] for v in listed_views),- "has_unscheduled": any(not v["suspended"] and not v["scheduled"] for v in listed_views),+ "sections": [section for section in sections if section["views"]],+ "summary": summary,"omitted_count": omitted_count,"views_url": f"{settings.SITE_URL}/project/{team_id}/models",},diff --git a/posthog/tasks/test/test_email.py b/posthog/tasks/test/test_email.pyindex 13964c92c1cc..1afb6815a442 100644--- a/posthog/tasks/test/test_email.py+++ b/posthog/tasks/test/test_email.py@@ -64,8 +64,9 @@ from products.batch_exports.backend.facade import testing as batch_exports_testifrom products.batch_exports.backend.facade.enums import BatchExportDestinationType, BatchExportRunStatusfrom products.cdp.backend.models.hog_functions.hog_function import HogFunctionfrom products.cdp.backend.models.plugin import Plugin, PluginConfig-from products.data_modeling.backend.facade.api import mark_node_suspended, sync_saved_query_to_dag+from products.data_modeling.backend.facade.api import mark_node_suspended, set_declared_target, sync_saved_query_to_dagfrom products.data_modeling.backend.facade.models import DataModelingJob, DataModelingJobEngine, DataWarehouseSavedQuery+from products.warehouse_sources.backend.facade.models import DataWarehouseTabledef _create_failed_batch_export_run(team_id: int) -> tuple[uuid.UUID, uuid.UUID]:@@ -2698,15 +2699,81 @@ class TestEmail(APIBaseTest, ClickhouseTestMixin):last_run_at=timezone.now() - dt.timedelta(hours=1),)+ # Pausing keeps `is_materialized` and the table but clears the cadence, so it can't retry.+ paused = DataWarehouseSavedQuery.objects.create(+ team=self.team,+ name="paused_orders_rollup",+ query={"query": "SELECT 1"},+ sync_frequency_interval=None,+ is_materialized=True,+ table=DataWarehouseTable.objects.create(team=self.team, name="paused_orders_rollup", format="Delta"),+ )+ # Tiered views keep their cadence on the node only, with the column empty.+ tiered = DataWarehouseSavedQuery.objects.create(+ team=self.team,+ name="tiered_daily_active_users",+ query={"query": "SELECT 1"},+ sync_frequency_interval=None,+ is_materialized=True,+ )+ tiered_node = sync_saved_query_to_dag(tiered)+ assert tiered_node is not None+ set_declared_target(tiered_node, dt.timedelta(hours=1))+ for sq in (paused, tiered):+ DataModelingJob.objects.create(+ team=self.team,+ saved_query=sq,+ status=DataModelingJob.Status.FAILED,+ error="Code: 241. DB::Exception: Memory limit (for query) exceeded",+ last_run_at=timezone.now() - dt.timedelta(hours=1),+ )++ # A success breaks the streak, so only the three failures after it count.+ ongoing = DataWarehouseSavedQuery.objects.create(+ team=self.team,+ name="nightly_revenue_rollup",+ query={"query": "SELECT 1"},+ sync_frequency_interval=dt.timedelta(days=1),+ is_materialized=True,+ )+ streak_started_at = timezone.now() - dt.timedelta(days=3)+ for status, run_at in [+ ("FAILED", timezone.now() - dt.timedelta(days=9)),+ ("COMPLETED", timezone.now() - dt.timedelta(days=6)),+ ("FAILED", streak_started_at),+ ("FAILED", timezone.now() - dt.timedelta(days=2)),+ ("FAILED", timezone.now() - dt.timedelta(hours=1)),+ ]:+ DataModelingJob.objects.create(+ team=self.team,+ saved_query=ongoing,+ status=getattr(DataModelingJob.Status, status),+ error="Code: 60. DB::Exception: Unknown table expression identifier",+ last_run_at=run_at,+ )+send_matview_failure_digest()assert len(mocked_email_messages) == 1- html = mocked_email_messages[0].html_body- for name, _ in failed_cases + unscheduled_cases:- assert name in html- assert html.count("Will retry") == len(failed_cases)- assert html.count("Not scheduled") == len(unscheduled_cases)- assert "action required" not in html+ message = mocked_email_messages[0]+ assert message.subject == (+ f"PostHog: 3 materialized views need action, 4 started failing, 1 is still failing in {self.team.name}"+ )+ sections = {s["key"]: [v["name"] for v in s["views"]] for s in message.properties["sections"]}+ assert list(sections) == ["action", "new", "ongoing"]+ assert sorted(sections["action"]) == sorted([*(name for name, _ in unscheduled_cases), paused.name])+ assert sorted(sections["new"]) == sorted([*(name for name, _ in failed_cases), tiered.name])+ assert sections["ongoing"] == ["nightly_revenue_rollup"]+ ongoing_row = next(v for v in message.properties["views"] if v["name"] == "nightly_revenue_rollup")+ assert ongoing_row["failed_runs"] == 3+ assert ongoing_row["failing_since"] == streak_started_at.strftime("%b %d, %H:%M UTC")++ html = message.html_body+ assert html.count("Not scheduled.") == len(unscheduled_cases)+ assert html.count("Paused.") == 1+ assert html.count("First failed") == len(failed_cases) + 1+ assert "3 failed runs" in html+ assert "select Resume" not in html@parameterized.expand([@@ -2948,16 +3015,15 @@ class TestEmail(APIBaseTest, ClickhouseTestMixin):assert len(mocked_email_messages) == 1views = mocked_email_messages[0].properties["views"]assert [(v["name"], v["suspended"]) for v in views] == expected_rows- assert mocked_email_messages[0].properties["has_suspended"] is expected_has_suspendedhtml = mocked_email_messages[0].html_body- assert "Will retry" in html+ assert "Started failing" in htmlif expected_has_suspended:- assert "action required" in html+ assert "select Resume" in htmlassert "Memory limit (for query) exceeded" in htmlassert "has been suspended after" not in htmlelse:- assert "action required" not in html+ assert "select Resume" not in htmlassert "suspended_view" not in htmldef test_send_matview_failure_digest_caps_the_rows_and_counts_the_rest(self, MockEmailMessage: MagicMock) -> None:diff --git a/posthog/templates/email/matview_failure_digest.html b/posthog/templates/email/matview_failure_digest.htmlindex b5d69594cfbe..f9af50e396ec 100644--- a/posthog/templates/email/matview_failure_digest.html+++ b/posthog/templates/email/matview_failure_digest.html@@ -1,87 +1,72 @@{% extends "email/base.html" %}-{% block preheader %}Materialized view failures in {{ team.name }}{% endblock %}+{% block preheader %}{{ summary }} in {{ team.name }}{% endblock %}{% block heading %}Materialized view failures{% endblock %}{% block section %}<p style="color: #374151;">- Some materialized views are failing in project <strong>{{ team.name }}</strong>. Their data may be out of date- until a run succeeds.- {% if has_suspended %}- Views marked <strong>suspended</strong> failed too many times in a row, so we stopped running them. Open the- view, fix the error, and run it once to start it again.- {% endif %}- {% if has_unscheduled %}- Views marked <strong>not scheduled</strong> are no longer materialized, so they will not run again until- you materialize them.- {% endif %}+ In project <strong>{{ team.name }}</strong>: {{ summary }}. Their data may be out of date until a run succeeds.</p>- <div style="background-color: #ffffff;- border: 1px solid #e5e7eb;- border-left: 4px solid {% if has_suspended %}#dc3545{% else %}#fd7e14{% endif %};- border-radius: 8px;- padding: 16px 20px;- margin-bottom: 12px">- <table width="100%"- cellpadding="0"- cellspacing="0"- border="0"- style="border-collapse: collapse">- {% for view in views %}- <tr>- <td align="left"- style="padding: 9px 8px 9px 0;- border-top: 1px solid #f3f4f6;- vertical-align: top;- text-align: left;- white-space: nowrap">- <a href="{{ view.url }}"- target="_blank"- style="text-decoration: none;- color: #111827;- font-weight: 600;- font-size: 0.9em">{{ view.name|truncatechars:40 }}</a>- <br />- <span style="color: #9ca3af; font-size: 0.75em;">{{ view.last_run_at }}</span>- </td>- <td align="left"- style="padding: 9px 8px;- border-top: 1px solid #f3f4f6;- vertical-align: top;- text-align: left;- color: #6b7280;- font-size: 0.85em;- word-break: break-word">{{ view.error|truncatechars:90 }}</td>- <td align="right"- style="padding: 9px 0 9px 8px;- border-top: 1px solid #f3f4f6;- vertical-align: top;- text-align: right;- white-space: nowrap">- {% if view.suspended %}- <span style="background-color: #fee2e2;- color: #b91c1c;- border-radius: 9999px;- padding: 2px 10px;- font-size: 0.75em;- font-weight: 600">Suspended, action required</span>- {% elif not view.scheduled %}- <span style="background-color: #fef3c7;- color: #92400e;- border-radius: 9999px;- padding: 2px 10px;- font-size: 0.75em;- font-weight: 600">Not scheduled</span>- {% else %}- <span style="background-color: #f3f4f6;- color: #6b7280;- border-radius: 9999px;- padding: 2px 10px;- font-size: 0.75em">Will retry</span>- {% endif %}- </td>- </tr>- {% endfor %}- </table>- </div>+ {% for section in sections %}+ <h3 style="color: #111827; font-size: 1em; margin: 20px 0 4px 0;">{{ section.title }} ({{ section.views|length }})</h3>+ <p style="color: #6b7280; font-size: 0.85em; margin: 0 0 8px 0;">{{ section.description }}</p>+ <div style="background-color: #ffffff;+ border: 1px solid #e5e7eb;+ border-left: 4px solid {{ section.color }};+ border-radius: 8px;+ padding: 8px 20px;+ margin-bottom: 12px">+ <table width="100%"+ cellpadding="0"+ cellspacing="0"+ border="0"+ style="border-collapse: collapse">+ {% for view in section.views %}+ <tr>+ <td align="left"+ style="padding: 9px 8px 9px 0;+ border-top: {% if forloop.first %}0{% else %}1px solid #f3f4f6{% endif %};+ vertical-align: top;+ text-align: left;+ white-space: nowrap">+ <a href="{{ view.url }}"+ target="_blank"+ style="text-decoration: none;+ color: #111827;+ font-weight: 600;+ font-size: 0.9em">{{ view.name|truncatechars:40 }}</a>+ <br />+ <span style="color: #9ca3af; font-size: 0.75em;">+ {% if section.key == "new" %}+ First failed {{ view.failing_since }}+ {% else %}+ Failing since {{ view.failing_since }}, {{ view.failed_runs }} failed run{{ view.failed_runs|pluralize }}+ {% endif %}+ </span>+ </td>+ <td align="left"+ style="padding: 9px 8px;+ border-top: {% if forloop.first %}0{% else %}1px solid #f3f4f6{% endif %};+ vertical-align: top;+ text-align: left;+ color: #6b7280;+ font-size: 0.85em;+ word-break: break-word">+ {{ view.error|truncatechars:90 }}+ {% if view.suspended %}+ <br />+ <span style="color: #b91c1c; font-size: 0.9em;">Suspended after repeated failures. Fix the error, then select Resume on the view.</span>+ {% elif view.schedule == "paused" %}+ <br />+ <span style="color: #92400e; font-size: 0.9em;">Paused. Choose a sync frequency in the view's materialization settings to restart it.</span>+ {% elif view.schedule == "reverted" %}+ <br />+ <span style="color: #92400e; font-size: 0.9em;">Not scheduled. Materialize the view again to restart it.</span>+ {% endif %}+ </td>+ </tr>+ {% endfor %}+ </table>+ </div>+ {% endfor %}{% if omitted_count %}<p style="font-size: 0.9em; color: #374151; margin-top: 8px;">…and <strong>{{ omitted_count }}</strong> more failing view{{ omitted_count|pluralize }}.