Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): retry missing read rollup days and gate deprecate (#130270)

Thiago Salvatore3fed10442c96
diff --git a/posthog/dags/tests/test_warehouse_object_reads_daily.py b/posthog/dags/tests/test_warehouse_object_reads_daily.py
index dba103710283..0bab172f400a 100644
--- a/posthog/dags/tests/test_warehouse_object_reads_daily.py
+++ b/posthog/dags/tests/test_warehouse_object_reads_daily.py
@@ -243,9 +243,43 @@ def test_rollup_of_a_day_without_query_log_rows_fails(cluster: ClickhouseCluster
run_rollup(cluster, date.fromisoformat(ROLLUP_START_DATE), succeeds=False)
-def test_schedule_runs_after_the_archive_day_closes() -> None:
- assert isinstance(warehouse_object_reads_daily_schedule, dagster.ScheduleDefinition)
- assert warehouse_object_reads_daily_schedule.cron_schedule == "0 7 * * *"
+CHECK_DAY = date(2026, 10, 8)
+
+
+def publish_days_except(instance: dagster.DagsterInstance, unpublished_days_ago: list[int]) -> None:
+ for days_ago in range(1, 11):
+ if days_ago in unpublished_days_ago:
+ continue
+ instance.create_run_for_job(
+ job_def=warehouse_object_reads_daily_job,
+ status=dagster.DagsterRunStatus.SUCCESS,
+ tags={"dagster/partition": (CHECK_DAY - timedelta(days=days_ago)).isoformat()},
+ )
+
+
+ [
+ ("newest_missing_day_first", [1, 4], False, "2026-10-07"),
+ ("older_missing_day", [4], False, "2026-10-04"),
+ ("missing_day_past_the_catch_up_window", [8], False, None),
+ ("while_a_rollup_runs", [1], True, None),
+ ]
+)
+def test_schedule_requests_the_newest_unpublished_day(
+ _name: str, unpublished_days_ago: list[int], rollup_running: bool, expected_partition: str | None
+) -> None:
+ with (
+ dagster.instance_for_test() as instance,
+ time_machine.travel(datetime.combine(CHECK_DAY, time(hour=7), tzinfo=UTC), tick=False),
+ ):
+ publish_days_except(instance, unpublished_days_ago)
+ if rollup_running:
+ instance.create_run_for_job(
+ job_def=warehouse_object_reads_daily_job, status=dagster.DagsterRunStatus.STARTED
+ )
+ result = warehouse_object_reads_daily_schedule(dagster.build_schedule_context(instance=instance))
+
+ assert (result.partition_key if isinstance(result, dagster.RunRequest) else None) == expected_partition
@pytest.mark.django_db
@@ -267,29 +301,29 @@ def test_rollup_fails_before_touching_staging_while_another_run_executes(cluster
@parameterized.expand(
[
- ("before_the_deadline", time(hour=9, minute=59), False, False, 0),
- ("published_day", time(hour=10), True, False, 0),
- ("missing_day", time(hour=10), False, False, 1),
- ("missing_day_slack_down", time(hour=10), False, True, 2),
+ ("before_the_deadline", time(hour=9, minute=59), [1], False, 0, None),
+ ("every_day_published", time(hour=10), [], False, 0, None),
+ ("missing_yesterday", time(hour=10), [1], False, 1, "2026-10-07"),
+ ("missing_older_day", time(hour=10), [4], False, 1, "2026-10-04"),
+ ("missing_day_slack_down", time(hour=10), [1], True, 2, "2026-10-07"),
]
)
@override_settings(CLOUD_DEPLOYMENT="US", DAGSTER_DOMAIN="dagster.example.com")
-def test_overdue_sensor_alerts_once_when_yesterday_is_not_published(
- _name: str, check_time: time, published: bool, slack_down: bool, expected_posts: int
+def test_overdue_sensor_alerts_once_a_day_about_unpublished_days(
+ _name: str,
+ check_time: time,
+ unpublished_days_ago: list[int],
+ slack_down: bool,
+ expected_posts: int,
+ expected_day: str | None,
) -> None:
- today = date(2026, 10, 8)
instance = dagster.DagsterInstance.ephemeral()
- if published:
- instance.create_run_for_job(
- job_def=warehouse_object_reads_daily_job,
- status=dagster.DagsterRunStatus.SUCCESS,
- tags={"dagster/partition": (today - timedelta(days=1)).isoformat()},
- )
+ publish_days_except(instance, unpublished_days_ago)
slack = MagicMock()
if slack_down:
slack.get_client.return_value.chat_postMessage.side_effect = ConnectionError
- with time_machine.travel(datetime.combine(today, check_time, tzinfo=UTC), tick=False):
+ with time_machine.travel(datetime.combine(CHECK_DAY, check_time, tzinfo=UTC), tick=False):
context = dagster.build_sensor_context(instance=instance, resources={"slack": slack})
warehouse_object_reads_daily_overdue_sensor(context)
warehouse_object_reads_daily_overdue_sensor(
@@ -300,4 +334,4 @@ def test_overdue_sensor_alerts_once_when_yesterday_is_not_published(
assert len(posts) == expected_posts
if expected_posts:
assert posts[0].kwargs["channel"] == "#alerts-data-modeling"
- assert "2026-10-07" in posts[0].kwargs["text"]
+ assert expected_day in posts[0].kwargs["text"]
diff --git a/posthog/dags/warehouse_object_reads_daily.py b/posthog/dags/warehouse_object_reads_daily.py
index 518b6ceae7a1..8bf662f0608e 100644
--- a/posthog/dags/warehouse_object_reads_daily.py
+++ b/posthog/dags/warehouse_object_reads_daily.py
@@ -39,7 +39,8 @@ SPILL_GROUP_BY_AFTER_BYTES = 2 * ONE_GIB
MAX_THREADS = 8
ROLLUP_NODE_ROLE = NodeRole.AUX
ROLLUP_START_DATE = "2026-09-17"
-SCHEDULE_HOUR_UTC = 7
+ATTEMPT_HOURS_UTC = (5, 6, 7)
+CATCH_UP_DAYS = 7
OVERDUE_AFTER_UTC = time(hour=10)
OVERDUE_CHECK_INTERVAL_SECONDS = 60 * 60
CONCURRENCY_TAG = {"warehouse_object_reads_backfill_concurrency": "warehouse_object_reads_v1"}
@@ -320,14 +321,6 @@ def warehouse_object_reads_daily_job() -> None:
rollup_warehouse_object_reads_for_day()
-warehouse_object_reads_daily_schedule = dagster.build_schedule_from_partitioned_job(
- warehouse_object_reads_daily_job,
- hour_of_day=SCHEDULE_HOUR_UTC,
- minute_of_hour=0,
- default_status=dagster.DefaultScheduleStatus.RUNNING,
-)
-
-
def has_published_day(instance: dagster.DagsterInstance, day: date) -> bool:
return bool(
instance.get_run_records(
@@ -341,6 +334,36 @@ def has_published_day(instance: dagster.DagsterInstance, day: date) -> bool:
)
+def unpublished_days(instance: dagster.DagsterInstance, today: date) -> list[date]:
+ first_day = max(date.fromisoformat(ROLLUP_START_DATE), today - timedelta(days=CATCH_UP_DAYS))
+ days = (first_day + timedelta(days=offset) for offset in range((today - first_day).days))
+ return [day for day in days if not has_published_day(instance, day)]
+
+
+ job=warehouse_object_reads_daily_job,
+ name=f"{warehouse_object_reads_daily_job.name}_schedule",
+ cron_schedule=f"0 {','.join(str(hour) for hour in ATTEMPT_HOURS_UTC)} * * *",
+ execution_timezone="UTC",
+ default_status=dagster.DefaultScheduleStatus.RUNNING,
+)
+def warehouse_object_reads_daily_schedule(
+ context: dagster.ScheduleEvaluationContext,
+) -> dagster.RunRequest | dagster.SkipReason:
+ active_runs = describe_runs(
+ context.instance,
+ (warehouse_object_reads_daily_job.name,),
+ created_after=datetime.now(UTC) - STUCK_RUN_AGE,
+ exclude_run_id="",
+ )
+ if active_runs:
+ return dagster.SkipReason(f"{'; '.join(active_runs)} is queued or executing.")
+ missing_days = unpublished_days(context.instance, datetime.now(UTC).date())
+ if not missing_days:
+ return dagster.SkipReason(f"Every day of the last {CATCH_UP_DAYS} days is published.")
+ return dagster.RunRequest(partition_key=missing_days[-1].isoformat())
+
+
@dagster.sensor(
minimum_interval_seconds=OVERDUE_CHECK_INTERVAL_SECONDS,
default_status=dagster.DefaultSensorStatus.RUNNING,
@@ -351,22 +374,26 @@ def warehouse_object_reads_daily_overdue_sensor(
now = datetime.now(UTC)
if now.time() < OVERDUE_AFTER_UTC:
return dagster.SkipReason(f"The rollup is not overdue before {OVERDUE_AFTER_UTC} UTC.")
- day = now.date() - timedelta(days=1)
- if context.cursor == day.isoformat():
- return dagster.SkipReason(f"Already alerted that {day} is missing.")
- if has_published_day(context.instance, day):
- return dagster.SkipReason(f"{day} is published.")
+ today = now.date()
+ if context.cursor == today.isoformat():
+ return dagster.SkipReason(f"Already alerted about missing days on {today}.")
+ missing_days = unpublished_days(context.instance, today)
+ if not missing_days:
+ return dagster.SkipReason(f"Every day of the last {CATCH_UP_DAYS} days is published.")
+ missing = ", ".join(day.isoformat() for day in missing_days)
if not settings.CLOUD_DEPLOYMENT:
- return dagster.SkipReason(f"{day} is missing, but alerts only go out in cloud deployments.")
+ return dagster.SkipReason(f"{missing} is missing, but alerts only go out in cloud deployments.")
+ attempt_times = ", ".join(f"{hour:02d}:00" for hour in ATTEMPT_HOURS_UTC)
job_url = f"https://{settings.DAGSTER_DOMAIN}/locations/clickhouse/jobs/{warehouse_object_reads_daily_job.name}"
message = (
- f":hourglass: `{warehouse_object_reads_daily_job.name}` has no successful run for {day} "
- f"by {OVERDUE_AFTER_UTC:%H:%M} UTC. Check that its schedule is running and that no run is stuck in the queue: "
+ f":hourglass: `{warehouse_object_reads_daily_job.name}` has no successful run for {missing} "
+ f"by {OVERDUE_AFTER_UTC:%H:%M} UTC. The schedule retries missing days at {attempt_times} UTC "
+ f"until they are {CATCH_UP_DAYS} days old. Check that its schedule is running and that no run is stuck in the queue: "
f"<{job_url}|View in Dagster>\nEnvironment: {settings.CLOUD_DEPLOYMENT}"
)
blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}]
channel = notification_channel_per_team[JobOwners.TEAM_DATA_MODELING.value]
if send_slack_alert(context, slack.get_client(), channel, blocks, message):
- context.update_cursor(day.isoformat())
+ context.update_cursor(today.isoformat())
return None
diff --git a/products/warehouse_suggestions/backend/logic/accept.py b/products/warehouse_suggestions/backend/logic/accept.py
index 52cd1df1bbd8..10c986066dd8 100644
--- a/products/warehouse_suggestions/backend/logic/accept.py
+++ b/products/warehouse_suggestions/backend/logic/accept.py
@@ -167,7 +167,7 @@ class DeprecateAcceptor(CertificationAcceptor):
) -> CreatedAsset:
if not request.options.delete_view:
return super().accept(suggestion, payload, request)
- if suggestion.evidence.get("days_with_data", 0) < RULES.deprecate.min_days_with_data_to_delete:
+ if suggestion.evidence.get("days_with_data", 0) < RULES.window_days:
raise AcceptOptionRefusedError(DELETE_VIEW_FIELD, DELETE_NEEDS_EVERY_DAY_OF_READS)
blocker = _delete_blocker(suggestion.team_id, suggestion.subject_id)
if blocker is not None:
diff --git a/products/warehouse_suggestions/backend/logic/candidates/deprecate.py b/products/warehouse_suggestions/backend/logic/candidates/deprecate.py
index d0b2212a36b7..028c16429766 100644
--- a/products/warehouse_suggestions/backend/logic/candidates/deprecate.py
+++ b/products/warehouse_suggestions/backend/logic/candidates/deprecate.py
@@ -26,12 +26,12 @@ class DeprecateCandidate(Candidate):
def evaluate(self, context: CandidateContext) -> CandidateResult:
rules = context.rules.deprecate
- if context.reads.days_with_data < rules.min_days_with_data:
+ if context.reads.days_with_data < context.rules.window_days:
return CandidateResult(
drafts=(),
rejections=(),
skipped_reason=(
- f"needs {rules.min_days_with_data} days of read data, has {context.reads.days_with_data}"
+ f"needs {context.rules.window_days} days of read data, has {context.reads.days_with_data}"
),
)
unread = [
diff --git a/products/warehouse_suggestions/backend/logic/rules.py b/products/warehouse_suggestions/backend/logic/rules.py
index 40a87f49e491..029927cfc7a4 100644
--- a/products/warehouse_suggestions/backend/logic/rules.py
+++ b/products/warehouse_suggestions/backend/logic/rules.py
@@ -87,8 +87,6 @@ class CertifyRules:
@frozen
class DeprecateRules:
- min_days_with_data: int
- min_days_with_data_to_delete: int
seconds_floor: float
bytes_floor: float
counts_background_reads: bool
@@ -212,8 +210,6 @@ RULES = Rules(
top_share=0.10, min_users=5, min_users_floor=2, min_user_share=0.5, min_days=20, min_surfaces=2
),
deprecate=DeprecateRules(
- min_days_with_data=27,
- min_days_with_data_to_delete=30,
seconds_floor=600.0,
bytes_floor=float(TERABYTE),
counts_background_reads=True,
diff --git a/products/warehouse_suggestions/backend/tests/test_candidates.py b/products/warehouse_suggestions/backend/tests/test_candidates.py
index ef15e0f5ae29..920b84d23975 100644
--- a/products/warehouse_suggestions/backend/tests/test_candidates.py
+++ b/products/warehouse_suggestions/backend/tests/test_candidates.py
@@ -349,11 +349,11 @@ class TestMaterializeRefreshMode(BaseTest):
class TestDeprecateCandidate(BaseTest):
@parameterized.expand(
[
- ("a_few_missing_days", 29, [VIEW_ID], None),
- ("too_many_missing_days", 26, [], "needs 27 days of read data, has 26"),
+ ("every_day", 30, [VIEW_ID], None),
+ ("one_missing_day", 29, [], "needs 30 days of read data, has 29"),
]
)
- def test_waits_for_most_of_a_window_of_read_data(
+ def test_waits_for_every_day_of_read_data(
self, _name: str, days_with_data: int, expected_subjects: list, expected_skip: str | None
) -> None:
candidate_context = context(