Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): group failure digest into new, ongoing and action sections (#121724)

Jovan Sakovicc98a6a21cbc3
diff --git a/posthog/tasks/email.py b/posthog/tasks/email.py
index edb748ced52d..aa386eb239ec 100644
--- a/posthog/tasks/email.py
+++ b/posthog/tasks/email.py
@@ -1,12 +1,14 @@
import uuid
import hashlib
import datetime
+import collections
from enum import Enum
from typing import Any, Literal, Optional, cast
from django.conf import settings
from django.db import transaction
-from django.db.models import OuterRef, Subquery
+from django.db.models import Count, OuterRef, Subquery, Value
+from django.db.models.functions import Coalesce
from django.utils import timezone
import structlog
@@ -60,6 +62,7 @@ from products.cdp.backend.models.hog_functions.hog_function import HogFunction
from products.cdp.backend.models.plugin import Plugin, PluginConfig
from products.conversations.backend.models import Ticket
from products.data_modeling.backend.facade.api import (
+ declared_targets_by_saved_query,
suspended_saved_query_ids_by_team,
suspension_state_for_saved_query,
)
@@ -1333,6 +1336,46 @@ def send_warehouse_destination_paused(
MAX_VIEWS_PER_DIGEST_EMAIL = 30
MAX_ERROR_CHARS = 255
+# Most urgent first: this order drives the email layout, the subject line and which rows the cap keeps.
+DIGEST_SECTIONS: dict[str, dict[str, Any]] = {
+ "action": {
+ "title": "Needs action",
+ "description": "These views stopped running. They won't update until you fix them.",
+ "color": "#dc3545",
+ "verbs": ("needs action", "need action"),
+ },
+ "new": {
+ "title": "Started failing",
+ "description": "These started failing in the last 24 hours. They retry on their next scheduled run, so a temporary problem can clear on its own.",
+ "color": "#fd7e14",
+ "verbs": ("started failing", "started failing"),
+ },
+ "ongoing": {
+ "title": "Still failing",
+ "description": "These have failed every run for more than a day. Retrying hasn't fixed them, so they likely need a change.",
+ "color": "#9ca3af",
+ "verbs": ("is still failing", "are still failing"),
+ },
+}
+DIGEST_SECTION_ORDER = tuple(DIGEST_SECTIONS)
+# Older than any job, so a view that never succeeded counts every failure in its streak.
+STREAK_FLOOR_BEFORE_ANY_RUN = datetime.datetime(1970, 1, 1, tzinfo=datetime.UTC)
+
+
+def _matview_digest_summary(counts: dict[str, int]) -> str:
+ """Count each section, e.g. "1 materialized view needs action, 3 started failing"."""
+ parts: list[str] = []
+ for key in DIGEST_SECTION_ORDER:
+ count = counts.get(key, 0)
+ if not count:
+ continue
+ verb = DIGEST_SECTIONS[key]["verbs"][count != 1]
+ if not parts:
+ noun = "materialized view" if count == 1 else "materialized views"
+ parts.append(f"{count} {noun} {verb}")
+ else:
+ parts.append(f"{count} {verb}")
+ return ", ".join(parts) or "Materialized view failures"
@shared_task(ignore_result=True)
@@ -1419,7 +1462,25 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],
return
all_ids = list(set(failed_query_ids + suspended_query_ids))
- queries = {str(sq.id): sq for sq in DataWarehouseSavedQuery.objects.filter(id__in=all_ids, team_id=team_id)}
+ clickhouse_jobs = DataModelingJob.objects.filter(
+ saved_query_id=OuterRef("id"), engine=DataModelingJobEngine.CLICKHOUSE, last_run_at__isnull=False
+ )
+ last_success_at = (
+ clickhouse_jobs.filter(status=DataModelingJob.Status.COMPLETED).order_by("-last_run_at").values("last_run_at")
+ )
+ streak_jobs = clickhouse_jobs.filter(status=DataModelingJob.Status.FAILED, last_run_at__gt=OuterRef("streak_floor"))
+ queries: dict[str, DataWarehouseSavedQuery] = {
+ str(sq.id): sq
+ for sq in DataWarehouseSavedQuery.objects.filter(id__in=all_ids, team_id=team_id).annotate(
+ streak_floor=Coalesce(Subquery(last_success_at[:1]), Value(STREAK_FLOOR_BEFORE_ANY_RUN)),
+ failing_since=Subquery(streak_jobs.order_by("last_run_at").values("last_run_at")[:1]),
+ failed_runs=Subquery(
+ streak_jobs.order_by().values("saved_query_id").annotate(n=Count("id")).values("n")[:1]
+ ),
+ )
+ }
+ new_failure_cutoff = timezone.now() - datetime.timedelta(hours=24)
+ declared_targets = declared_targets_by_saved_query(team_id, all_ids)
latest_jobs: dict[str, DataModelingJob] = {}
for latest_job in (
@@ -1447,6 +1508,21 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],
if len(error) > MAX_ERROR_CHARS:
error = error[: MAX_ERROR_CHARS - 3] + "..."
run_at = (job.last_run_at if job else None) or sq.last_run_at
+ # `is_materialized` stays True after a pause, so read the cadence and the table instead.
+ # Pausing clears the cadence and keeps the table; reverting clears both.
+ if qid in declared_targets or sq.sync_frequency_interval is not None:
+ schedule = "scheduled"
+ elif sq.table_id is not None:
+ schedule = "paused"
+ else:
+ schedule = "reverted"
+ failing_since: datetime.datetime | None = getattr(sq, "failing_since", None) or run_at
+ if suspended or schedule != "scheduled":
+ section = "action"
+ elif failing_since is None or failing_since >= new_failure_cutoff:
+ section = "new"
+ else:
+ section = "ongoing"
views.append(
{
"id": str(sq.id),
@@ -1454,11 +1530,11 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],
"error": error,
"last_run_at": run_at.strftime("%b %d, %H:%M UTC") if run_at else "Unknown",
"last_run_at_ts": run_at.timestamp() if run_at else 0,
+ "failing_since": failing_since.strftime("%b %d, %H:%M UTC") if failing_since else "Unknown",
+ "failed_runs": getattr(sq, "failed_runs", None) or 1,
+ "section": section,
"suspended": suspended,
- # Reverting clears the schedule, so promising a retry here would be false. `is not
- # False` keeps a never-written flag on the retrying side, as `exclude(deleted=True)`
- # does above.
- "scheduled": sq.is_materialized is not False,
+ "schedule": schedule,
"url": f"{settings.SITE_URL}/project/{team_id}/sql?open_view={sq.id}",
}
)
@@ -1467,8 +1543,14 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],
logger.warning("No failed or suspended views found")
return
- # Rows needing action outrank rows that heal themselves, so the cap keeps them.
- views.sort(key=lambda v: (not v["suspended"], v["scheduled"], -cast(float, v["last_run_at_ts"])))
+ # Rows needing action outrank new failures, which outrank old ones, so the cap keeps them.
+ views.sort(
+ key=lambda v: (
+ DIGEST_SECTION_ORDER.index(v["section"]),
+ not v["suspended"],
+ -cast(float, v["last_run_at_ts"]),
+ )
+ )
for v in views:
v.pop("last_run_at_ts", None)
@@ -1479,15 +1561,27 @@ def send_team_matview_failure_digest(team_id: int, failed_query_ids: list[str],
omitted_count = max(0, len(visible_views) - MAX_VIEWS_PER_DIGEST_EMAIL)
listed_views = visible_views[:MAX_VIEWS_PER_DIGEST_EMAIL]
+ # Count before the cap, so the subject reflects every failing view the reader can see.
+ summary = _matview_digest_summary(collections.Counter(v["section"] for v in visible_views))
+ sections = [
+ {
+ "key": key,
+ "title": DIGEST_SECTIONS[key]["title"],
+ "description": DIGEST_SECTIONS[key]["description"],
+ "color": DIGEST_SECTIONS[key]["color"],
+ "views": [v for v in listed_views if v["section"] == key],
+ }
+ for key in DIGEST_SECTION_ORDER
+ ]
message = EmailMessage(
campaign_key=campaign_key,
- subject=f"PostHog: Materialized view failures in {team.name}",
+ subject=f"PostHog: {summary} in {team.name}",
template_name="matview_failure_digest",
template_context={
"team": team,
"views": listed_views,
- "has_suspended": any(v["suspended"] for v in listed_views),
- "has_unscheduled": any(not v["suspended"] and not v["scheduled"] for v in listed_views),
+ "sections": [section for section in sections if section["views"]],
+ "summary": summary,
"omitted_count": omitted_count,
"views_url": f"{settings.SITE_URL}/project/{team_id}/models",
},
diff --git a/posthog/tasks/test/test_email.py b/posthog/tasks/test/test_email.py
index 13964c92c1cc..1afb6815a442 100644
--- a/posthog/tasks/test/test_email.py
+++ b/posthog/tasks/test/test_email.py
@@ -64,8 +64,9 @@ from products.batch_exports.backend.facade import testing as batch_exports_testi
from products.batch_exports.backend.facade.enums import BatchExportDestinationType, BatchExportRunStatus
from products.cdp.backend.models.hog_functions.hog_function import HogFunction
from products.cdp.backend.models.plugin import Plugin, PluginConfig
-from products.data_modeling.backend.facade.api import mark_node_suspended, sync_saved_query_to_dag
+from products.data_modeling.backend.facade.api import mark_node_suspended, set_declared_target, sync_saved_query_to_dag
from products.data_modeling.backend.facade.models import DataModelingJob, DataModelingJobEngine, DataWarehouseSavedQuery
+from products.warehouse_sources.backend.facade.models import DataWarehouseTable
def _create_failed_batch_export_run(team_id: int) -> tuple[uuid.UUID, uuid.UUID]:
@@ -2698,15 +2699,81 @@ class TestEmail(APIBaseTest, ClickhouseTestMixin):
last_run_at=timezone.now() - dt.timedelta(hours=1),
)
+ # Pausing keeps `is_materialized` and the table but clears the cadence, so it can't retry.
+ paused = DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="paused_orders_rollup",
+ query={"query": "SELECT 1"},
+ sync_frequency_interval=None,
+ is_materialized=True,
+ table=DataWarehouseTable.objects.create(team=self.team, name="paused_orders_rollup", format="Delta"),
+ )
+ # Tiered views keep their cadence on the node only, with the column empty.
+ tiered = DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="tiered_daily_active_users",
+ query={"query": "SELECT 1"},
+ sync_frequency_interval=None,
+ is_materialized=True,
+ )
+ tiered_node = sync_saved_query_to_dag(tiered)
+ assert tiered_node is not None
+ set_declared_target(tiered_node, dt.timedelta(hours=1))
+ for sq in (paused, tiered):
+ DataModelingJob.objects.create(
+ team=self.team,
+ saved_query=sq,
+ status=DataModelingJob.Status.FAILED,
+ error="Code: 241. DB::Exception: Memory limit (for query) exceeded",
+ last_run_at=timezone.now() - dt.timedelta(hours=1),
+ )
+
+ # A success breaks the streak, so only the three failures after it count.
+ ongoing = DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="nightly_revenue_rollup",
+ query={"query": "SELECT 1"},
+ sync_frequency_interval=dt.timedelta(days=1),
+ is_materialized=True,
+ )
+ streak_started_at = timezone.now() - dt.timedelta(days=3)
+ for status, run_at in [
+ ("FAILED", timezone.now() - dt.timedelta(days=9)),
+ ("COMPLETED", timezone.now() - dt.timedelta(days=6)),
+ ("FAILED", streak_started_at),
+ ("FAILED", timezone.now() - dt.timedelta(days=2)),
+ ("FAILED", timezone.now() - dt.timedelta(hours=1)),
+ ]:
+ DataModelingJob.objects.create(
+ team=self.team,
+ saved_query=ongoing,
+ status=getattr(DataModelingJob.Status, status),
+ error="Code: 60. DB::Exception: Unknown table expression identifier",
+ last_run_at=run_at,
+ )
+
send_matview_failure_digest()
assert len(mocked_email_messages) == 1
- html = mocked_email_messages[0].html_body
- for name, _ in failed_cases + unscheduled_cases:
- assert name in html
- assert html.count("Will retry") == len(failed_cases)
- assert html.count("Not scheduled") == len(unscheduled_cases)
- assert "action required" not in html
+ message = mocked_email_messages[0]
+ assert message.subject == (
+ f"PostHog: 3 materialized views need action, 4 started failing, 1 is still failing in {self.team.name}"
+ )
+ sections = {s["key"]: [v["name"] for v in s["views"]] for s in message.properties["sections"]}
+ assert list(sections) == ["action", "new", "ongoing"]
+ assert sorted(sections["action"]) == sorted([*(name for name, _ in unscheduled_cases), paused.name])
+ assert sorted(sections["new"]) == sorted([*(name for name, _ in failed_cases), tiered.name])
+ assert sections["ongoing"] == ["nightly_revenue_rollup"]
+ ongoing_row = next(v for v in message.properties["views"] if v["name"] == "nightly_revenue_rollup")
+ assert ongoing_row["failed_runs"] == 3
+ assert ongoing_row["failing_since"] == streak_started_at.strftime("%b %d, %H:%M UTC")
+
+ html = message.html_body
+ assert html.count("Not scheduled.") == len(unscheduled_cases)
+ assert html.count("Paused.") == 1
+ assert html.count("First failed") == len(failed_cases) + 1
+ assert "3 failed runs" in html
+ assert "select Resume" not in html
@parameterized.expand(
[
@@ -2948,16 +3015,15 @@ class TestEmail(APIBaseTest, ClickhouseTestMixin):
assert len(mocked_email_messages) == 1
views = mocked_email_messages[0].properties["views"]
assert [(v["name"], v["suspended"]) for v in views] == expected_rows
- assert mocked_email_messages[0].properties["has_suspended"] is expected_has_suspended
html = mocked_email_messages[0].html_body
- assert "Will retry" in html
+ assert "Started failing" in html
if expected_has_suspended:
- assert "action required" in html
+ assert "select Resume" in html
assert "Memory limit (for query) exceeded" in html
assert "has been suspended after" not in html
else:
- assert "action required" not in html
+ assert "select Resume" not in html
assert "suspended_view" not in html
def test_send_matview_failure_digest_caps_the_rows_and_counts_the_rest(self, MockEmailMessage: MagicMock) -> None:
diff --git a/posthog/templates/email/matview_failure_digest.html b/posthog/templates/email/matview_failure_digest.html
index b5d69594cfbe..f9af50e396ec 100644
--- a/posthog/templates/email/matview_failure_digest.html
+++ b/posthog/templates/email/matview_failure_digest.html
@@ -1,87 +1,72 @@
{% extends "email/base.html" %}
-{% block preheader %}Materialized view failures in {{ team.name }}{% endblock %}
+{% block preheader %}{{ summary }} in {{ team.name }}{% endblock %}
{% block heading %}Materialized view failures{% endblock %}
{% block section %}
<p style="color: #374151;">
- Some materialized views are failing in project <strong>{{ team.name }}</strong>. Their data may be out of date
- until a run succeeds.
- {% if has_suspended %}
- Views marked <strong>suspended</strong> failed too many times in a row, so we stopped running them. Open the
- view, fix the error, and run it once to start it again.
- {% endif %}
- {% if has_unscheduled %}
- Views marked <strong>not scheduled</strong> are no longer materialized, so they will not run again until
- you materialize them.
- {% endif %}
+ In project <strong>{{ team.name }}</strong>: {{ summary }}. Their data may be out of date until a run succeeds.
</p>
- <div style="background-color: #ffffff;
- border: 1px solid #e5e7eb;
- border-left: 4px solid {% if has_suspended %}#dc3545{% else %}#fd7e14{% endif %};
- border-radius: 8px;
- padding: 16px 20px;
- margin-bottom: 12px">
- <table width="100%"
- cellpadding="0"
- cellspacing="0"
- border="0"
- style="border-collapse: collapse">
- {% for view in views %}
- <tr>
- <td align="left"
- style="padding: 9px 8px 9px 0;
- border-top: 1px solid #f3f4f6;
- vertical-align: top;
- text-align: left;
- white-space: nowrap">
- <a href="{{ view.url }}"
- target="_blank"
- style="text-decoration: none;
- color: #111827;
- font-weight: 600;
- font-size: 0.9em">{{ view.name|truncatechars:40 }}</a>
- <br />
- <span style="color: #9ca3af; font-size: 0.75em;">{{ view.last_run_at }}</span>
- </td>
- <td align="left"
- style="padding: 9px 8px;
- border-top: 1px solid #f3f4f6;
- vertical-align: top;
- text-align: left;
- color: #6b7280;
- font-size: 0.85em;
- word-break: break-word">{{ view.error|truncatechars:90 }}</td>
- <td align="right"
- style="padding: 9px 0 9px 8px;
- border-top: 1px solid #f3f4f6;
- vertical-align: top;
- text-align: right;
- white-space: nowrap">
- {% if view.suspended %}
- <span style="background-color: #fee2e2;
- color: #b91c1c;
- border-radius: 9999px;
- padding: 2px 10px;
- font-size: 0.75em;
- font-weight: 600">Suspended, action required</span>
- {% elif not view.scheduled %}
- <span style="background-color: #fef3c7;
- color: #92400e;
- border-radius: 9999px;
- padding: 2px 10px;
- font-size: 0.75em;
- font-weight: 600">Not scheduled</span>
- {% else %}
- <span style="background-color: #f3f4f6;
- color: #6b7280;
- border-radius: 9999px;
- padding: 2px 10px;
- font-size: 0.75em">Will retry</span>
- {% endif %}
- </td>
- </tr>
- {% endfor %}
- </table>
- </div>
+ {% for section in sections %}
+ <h3 style="color: #111827; font-size: 1em; margin: 20px 0 4px 0;">{{ section.title }} ({{ section.views|length }})</h3>
+ <p style="color: #6b7280; font-size: 0.85em; margin: 0 0 8px 0;">{{ section.description }}</p>
+ <div style="background-color: #ffffff;
+ border: 1px solid #e5e7eb;
+ border-left: 4px solid {{ section.color }};
+ border-radius: 8px;
+ padding: 8px 20px;
+ margin-bottom: 12px">
+ <table width="100%"
+ cellpadding="0"
+ cellspacing="0"
+ border="0"
+ style="border-collapse: collapse">
+ {% for view in section.views %}
+ <tr>
+ <td align="left"
+ style="padding: 9px 8px 9px 0;
+ border-top: {% if forloop.first %}0{% else %}1px solid #f3f4f6{% endif %};
+ vertical-align: top;
+ text-align: left;
+ white-space: nowrap">
+ <a href="{{ view.url }}"
+ target="_blank"
+ style="text-decoration: none;
+ color: #111827;
+ font-weight: 600;
+ font-size: 0.9em">{{ view.name|truncatechars:40 }}</a>
+ <br />
+ <span style="color: #9ca3af; font-size: 0.75em;">
+ {% if section.key == "new" %}
+ First failed {{ view.failing_since }}
+ {% else %}
+ Failing since {{ view.failing_since }}, {{ view.failed_runs }} failed run{{ view.failed_runs|pluralize }}
+ {% endif %}
+ </span>
+ </td>
+ <td align="left"
+ style="padding: 9px 8px;
+ border-top: {% if forloop.first %}0{% else %}1px solid #f3f4f6{% endif %};
+ vertical-align: top;
+ text-align: left;
+ color: #6b7280;
+ font-size: 0.85em;
+ word-break: break-word">
+ {{ view.error|truncatechars:90 }}
+ {% if view.suspended %}
+ <br />
+ <span style="color: #b91c1c; font-size: 0.9em;">Suspended after repeated failures. Fix the error, then select Resume on the view.</span>
+ {% elif view.schedule == "paused" %}
+ <br />
+ <span style="color: #92400e; font-size: 0.9em;">Paused. Choose a sync frequency in the view's materialization settings to restart it.</span>
+ {% elif view.schedule == "reverted" %}
+ <br />
+ <span style="color: #92400e; font-size: 0.9em;">Not scheduled. Materialize the view again to restart it.</span>
+ {% endif %}
+ </td>
+ </tr>
+ {% endfor %}
+ </table>
+ </div>
+ {% endfor %}
{% if omitted_count %}
<p style="font-size: 0.9em; color: #374151; margin-top: 8px;">
&hellip;and <strong>{{ omitted_count }}</strong> more failing view{{ omitted_count|pluralize }}.