Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): list a paused schema in one failure digest only (#124204)

Tom Owersf0326edfafb3
diff --git a/products/data_warehouse/backend/logic/external_data_source/notifications.py b/products/data_warehouse/backend/logic/external_data_source/notifications.py
index 1be8f3e3a738..98af24cc1c63 100644
--- a/products/data_warehouse/backend/logic/external_data_source/notifications.py
+++ b/products/data_warehouse/backend/logic/external_data_source/notifications.py
@@ -24,13 +24,6 @@ logger = structlog.get_logger(__name__)
# notified — the email communicates them in aggregate via the "+N more" line.
MAX_SCHEMAS_PER_DIGEST_EMAIL = 30
-# Re-notify a schema that stays failed this long after its last email. A paused
-# schema never runs again, so it produces no newer failed job to re-trigger the
-# digest — without this it gets exactly one email, then silence for weeks while
-# its data goes stale. The catch-up sweep runs daily, so it picks the schema up
-# the first day past this age and stamps a fresh time, spacing reminders evenly.
-RENOTIFY_STILL_FAILING_AFTER = dt.timedelta(days=7)
-
# A schema the user switched off is not a failure to chase: it will not run again, and nobody
# asked it to. Its last status stays Failed for the syncs UI, so the digest cannot read status
# alone. `auto_disabled_at` marks the other way syncing stops, where PostHog halted the schema
@@ -40,31 +33,22 @@ RENOTIFY_STILL_FAILING_AFTER = dt.timedelta(days=7)
SCHEMA_STOPPED_BY_USER = Q(should_sync=False, auto_disabled_at__isnull=True)
-def get_team_ids_with_recent_sync_failures(
- lookback: dt.timedelta = dt.timedelta(hours=26),
- renotify_after: dt.timedelta = RENOTIFY_STILL_FAILING_AFTER,
-) -> list[int]:
+def get_team_ids_with_recent_sync_failures(lookback: dt.timedelta = dt.timedelta(hours=26)) -> list[int]:
"""Teams with still-failing schemas that need a digest.
- Powers the daily catch-up digest. It flushes two groups:
-
- - Failures that the one-email-per-day block swallowed. A failure counts only
- if it is newer than the schema's `last_error_notified_at` stamp, so failures
- already covered by an earlier digest do not trigger a duplicate.
- - Schemas that stay failed but produce no newer failed run — a paused schema
- never syncs again. Their last email is older than `renotify_after`, so they
- get a reminder instead of silence.
-
- Schemas the user switched off are left out entirely, per `SCHEMA_STOPPED_BY_USER`.
+ Powers the daily catch-up digest. It flushes failures that the one-email-per-day
+ block swallowed. A failure counts only if it is newer than the schema's
+ `last_error_notified_at` stamp, so failures already covered by an earlier digest
+ do not trigger a duplicate. A paused schema never runs again, so after its first
+ digest it stays out until a new run fails. Schemas the user switched off are left
+ out entirely, per `SCHEMA_STOPPED_BY_USER`.
The lookback exceeds the 24h digest day on purpose: a failure just after the
10:00 UTC rollover, blocked because that digest day's email already went out,
is 24h15m+ old by the next catch-up run — a 24h lookback would drop it forever
for paused schemas. The stamp check above keeps the wider window duplicate-free.
"""
- now = dt.datetime.now(dt.UTC)
- cutoff = now - lookback
- renotify_cutoff = now - renotify_after
+ cutoff = dt.datetime.now(dt.UTC) - lookback
# Drive from the schema side: the jobs table grows with every sync run and has
# no index on (status, finished_at), so starting there would seq-scan it daily.
# Schemas are one row each, and their jobs are reachable via the schema_id FK index.
@@ -78,21 +62,18 @@ def get_team_ids_with_recent_sync_failures(
.exclude(source__deleted=True)
.exclude(SCHEMA_STOPPED_BY_USER)
.filter(status=ExternalDataSchemaStatus.FAILED)
- .filter(Exists(unnotified_failed_job) | Q(last_error_notified_at__lt=renotify_cutoff))
+ .filter(Exists(unnotified_failed_job))
.values_list("team_id", flat=True)
.distinct()
)
-def notify_external_data_sync_failures(
- team_id: int, renotify_after: dt.timedelta = RENOTIFY_STILL_FAILING_AFTER
-) -> None:
+def notify_external_data_sync_failures(team_id: int) -> None:
"""Email the team a digest of failing external data schemas that need a notification.
- A schema is listed when it was never notified, has a failed run newer than its last
- notification, or has stayed failed for longer than `renotify_after` since its last
- email. The last case reminds the team about a paused schema, which never runs again
- and so produces no newer failed run to re-trigger the digest on its own. Schemas of a
+ A schema is listed when it was never notified, or has a failed run newer than its last
+ notification. A paused schema never runs again, so it is listed once and then stays out
+ of later digests until someone turns syncing back on and a new run fails. Schemas of a
deleted source, and schemas the user switched off (see `SCHEMA_STOPPED_BY_USER`), are
excluded entirely. Runs inside the digest Celery task; exceptions are swallowed so a
notification problem never crash-loops the task. Throttling to one email per team per
@@ -100,7 +81,6 @@ def notify_external_data_sync_failures(
scheduling this for every failed job is safe.
"""
try:
- renotify_cutoff = dt.datetime.now(dt.UTC) - renotify_after
newer_failed_job = ExternalDataJob.objects.filter(
schema_id=OuterRef("id"),
status=ExternalDataJobStatus.FAILED,
@@ -111,11 +91,7 @@ def notify_external_data_sync_failures(
.exclude(source__deleted=True)
.exclude(SCHEMA_STOPPED_BY_USER)
.filter(team_id=team_id, status=ExternalDataSchemaStatus.FAILED)
- .filter(
- Q(last_error_notified_at__isnull=True)
- | Q(last_error_notified_at__lt=renotify_cutoff)
- | Exists(newer_failed_job)
- )
+ .filter(Q(last_error_notified_at__isnull=True) | Exists(newer_failed_job))
.select_related("source")
.order_by("name")
)
diff --git a/products/data_warehouse/backend/logic/external_data_source/test/test_notifications.py b/products/data_warehouse/backend/logic/external_data_source/test/test_notifications.py
index cf42bed816e0..f881fcf0756a 100644
--- a/products/data_warehouse/backend/logic/external_data_source/test/test_notifications.py
+++ b/products/data_warehouse/backend/logic/external_data_source/test/test_notifications.py
@@ -317,31 +317,38 @@ class TestNotifyExternalDataSyncFailures:
assert mock_sender.called == expected_sent
@pytest.mark.parametrize(
- "notified_age,expected_sent",
+ "should_sync,auto_disabled,sync_type_config,notified_age",
[
- (dt.timedelta(days=8), True), # still failing past the window -> remind again
- (dt.timedelta(days=6), False), # notified recently, no new run -> stay silent
+ (False, True, {}, dt.timedelta(days=8)),
+ (False, True, {}, dt.timedelta(days=90)),
+ (True, False, {"cdc_broken": {"reason": "auto_dropped_critical_lag"}}, dt.timedelta(days=8)),
],
)
- def test_renotifies_still_failing_schema_without_a_newer_run(self, notified_age, expected_sent):
- # A paused schema never runs again, so it produces no newer failed job. It must
- # still get a reminder once its last email ages past the re-notify window.
+ def test_paused_schema_is_listed_only_once(self, should_sync, auto_disabled, sync_type_config, notified_age):
+ # A paused schema never runs again, so once a digest has listed it, later digests stay
+ # silent about it however long it stays failed.
team, source = _create_team_and_source()
- ExternalDataSchema.objects.create(
+ notified_at = dt.datetime.now(dt.UTC) - notified_age
+ schema = ExternalDataSchema.objects.create(
name="Charge",
team=team,
source=source,
status=ExternalDataSchema.Status.FAILED,
- should_sync=False,
- auto_disabled_at=dt.datetime.now(dt.UTC) - notified_age,
+ should_sync=should_sync,
+ auto_disabled_at=notified_at if auto_disabled else None,
+ sync_type_config=sync_type_config,
latest_error="Invalid API key",
- last_error_notified_at=dt.datetime.now(dt.UTC) - notified_age,
+ last_error_notified_at=notified_at,
+ )
+ job = ExternalDataJob.objects.create(
+ team=team, pipeline=source, schema=schema, status=ExternalDataJob.Status.FAILED
)
+ ExternalDataJob.objects.filter(id=job.id).update(finished_at=notified_at - dt.timedelta(minutes=5))
with patch(SENDER_PATH) as mock_sender:
notify_external_data_sync_failures(team.pk)
- assert mock_sender.called == expected_sent
+ mock_sender.assert_not_called()
class TestGetTeamIdsWithRecentSyncFailures:
@@ -436,19 +443,19 @@ class TestGetTeamIdsWithRecentSyncFailures:
assert get_team_ids_with_recent_sync_failures() == [team.pk]
- def test_includes_still_failing_schema_past_the_renotify_window(self):
- # PostHog halted this schema itself, so it never runs again. No recent failed run
- # (30h stale) and the last email is 8 days old: the team would drop out forever
- # without the re-notify path.
- team = self._create_schema_with_job(
+ @pytest.mark.parametrize("notified_age", [dt.timedelta(days=8), dt.timedelta(days=90)])
+ def test_excludes_paused_schema_already_notified(self, notified_age):
+ # PostHog halted this schema itself, so it never runs again. Its one digest already went
+ # out, so the catch-up sweep must not pick it up again however old that email is.
+ self._create_schema_with_job(
schema_status=ExternalDataSchema.Status.FAILED,
- job_finished_at=dt.datetime.now(dt.UTC) - dt.timedelta(hours=30),
- last_error_notified_at=dt.datetime.now(dt.UTC) - dt.timedelta(days=8),
+ job_finished_at=dt.datetime.now(dt.UTC) - notified_age - dt.timedelta(minutes=5),
+ last_error_notified_at=dt.datetime.now(dt.UTC) - notified_age,
should_sync=False,
- auto_disabled_at=dt.datetime.now(dt.UTC) - dt.timedelta(days=9),
+ auto_disabled_at=dt.datetime.now(dt.UTC) - notified_age,
)
- assert get_team_ids_with_recent_sync_failures() == [team.pk]
+ assert get_team_ids_with_recent_sync_failures() == []
def test_only_returns_qualifying_teams(self):
qualifying = self._create_schema_with_job(
diff --git a/products/warehouse_sources/backend/stalled_schedules.py b/products/warehouse_sources/backend/stalled_schedules.py
index f3c7d9d4b063..b305abab98bc 100644
--- a/products/warehouse_sources/backend/stalled_schedules.py
+++ b/products/warehouse_sources/backend/stalled_schedules.py
@@ -308,13 +308,10 @@ def repair_stalled_schema(stalled: StalledSchema) -> bool:
REPAIRED_SCHEMA_ERROR,
logger,
)
- # get_team_ids_with_recent_sync_failures renotifies a FAILED schema once
- # last_error_notified_at is more than RENOTIFY_STILL_FAILING_AFTER old, with no newer
- # failed job required. A schema that failed long ago, was notified, and has since been
- # healthy for a while still carries that old stamp, so this repaint would otherwise read
- # as "still failing" and send the failure digest instead of the informational message
- # above. Stamping it here starts that window fresh; a genuinely new failure still notifies,
- # because its job's finished_at is newer than this stamp.
+ # The failure digest lists a FAILED schema that has no notification stamp, so without
+ # one this repaint would read as a sync failure in the team's next digest instead of
+ # the informational message above. A genuinely new failure still notifies, because its
+ # job's finished_at is newer than this stamp.
ExternalDataSchema.objects.filter(id=schema.id, team_id=schema.team_id).update(
last_error_notified_at=timezone.now()
)
diff --git a/products/warehouse_sources/backend/tests/test_stalled_schedules.py b/products/warehouse_sources/backend/tests/test_stalled_schedules.py
index 0ac7cdc75cb2..1cc8581c0bca 100644
--- a/products/warehouse_sources/backend/tests/test_stalled_schedules.py
+++ b/products/warehouse_sources/backend/tests/test_stalled_schedules.py
@@ -203,12 +203,14 @@ class TestStalledSchedules(BaseTest):
assert mock_sync.call_args.kwargs["trigger_immediately"] is False
assert mock_sync.call_args.kwargs["should_sync"] is True
- def test_repair_stamps_last_error_notified_at_to_avoid_a_stale_renotify(self) -> None:
- # get_team_ids_with_recent_sync_failures renotifies once last_error_notified_at is more
- # than seven days old, with no newer failed job required. Without a fresh stamp here, a
- # schema that failed and was notified long ago, then recovered, would have this repaint
- # read as "still failing" and send the failure digest instead of the informational message.
- old_notification = timezone.now() - timedelta(days=30)
+ @parameterized.expand([("never_notified", None), ("notified_long_ago", timedelta(days=30))])
+ def test_repair_stamps_last_error_notified_at_to_keep_the_repaint_out_of_the_digest(
+ self, _name: str, notified_ago: timedelta | None
+ ) -> None:
+ # The failure digest lists a FAILED schema with no notification stamp. Without a fresh
+ # stamp here, this repaint would read as a sync failure in the next digest instead of
+ # the informational message.
+ old_notification = timezone.now() - notified_ago if notified_ago is not None else None
schema = self._schema(
synced_ago=timedelta(days=5),
status=ExternalDataSchema.Status.RUNNING,
@@ -221,7 +223,8 @@ class TestStalledSchedules(BaseTest):
schema.refresh_from_db()
assert schema.last_error_notified_at is not None
- assert schema.last_error_notified_at > old_notification
+ if old_notification is not None:
+ assert schema.last_error_notified_at > old_notification
def test_repair_skips_a_schema_disabled_since_it_was_discovered(self) -> None:
# The confirmation prompt in the management command alone can put minutes between