:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): retry missing read rollup days and gate deprecate (#130270)
Thiago Salvatore3fed10442c96
diff --git a/posthog/dags/tests/test_warehouse_object_reads_daily.py b/posthog/dags/tests/test_warehouse_object_reads_daily.pyindex dba103710283..0bab172f400a 100644--- a/posthog/dags/tests/test_warehouse_object_reads_daily.py+++ b/posthog/dags/tests/test_warehouse_object_reads_daily.py@@ -243,9 +243,43 @@ def test_rollup_of_a_day_without_query_log_rows_fails(cluster: ClickhouseClusterrun_rollup(cluster, date.fromisoformat(ROLLUP_START_DATE), succeeds=False)-def test_schedule_runs_after_the_archive_day_closes() -> None:- assert isinstance(warehouse_object_reads_daily_schedule, dagster.ScheduleDefinition)- assert warehouse_object_reads_daily_schedule.cron_schedule == "0 7 * * *"+CHECK_DAY = date(2026, 10, 8)+++def publish_days_except(instance: dagster.DagsterInstance, unpublished_days_ago: list[int]) -> None:+ for days_ago in range(1, 11):+ if days_ago in unpublished_days_ago:+ continue+ instance.create_run_for_job(+ job_def=warehouse_object_reads_daily_job,+ status=dagster.DagsterRunStatus.SUCCESS,+ tags={"dagster/partition": (CHECK_DAY - timedelta(days=days_ago)).isoformat()},+ )+++ [+ ("newest_missing_day_first", [1, 4], False, "2026-10-07"),+ ("older_missing_day", [4], False, "2026-10-04"),+ ("missing_day_past_the_catch_up_window", [8], False, None),+ ("while_a_rollup_runs", [1], True, None),+ ]+)+def test_schedule_requests_the_newest_unpublished_day(+ _name: str, unpublished_days_ago: list[int], rollup_running: bool, expected_partition: str | None+) -> None:+ with (+ dagster.instance_for_test() as instance,+ time_machine.travel(datetime.combine(CHECK_DAY, time(hour=7), tzinfo=UTC), tick=False),+ ):+ publish_days_except(instance, unpublished_days_ago)+ if rollup_running:+ instance.create_run_for_job(+ job_def=warehouse_object_reads_daily_job, status=dagster.DagsterRunStatus.STARTED+ )+ result = warehouse_object_reads_daily_schedule(dagster.build_schedule_context(instance=instance))++ assert (result.partition_key if isinstance(result, dagster.RunRequest) else None) == expected_partition@pytest.mark.django_db@@ -267,29 +301,29 @@ def test_rollup_fails_before_touching_staging_while_another_run_executes(cluster@parameterized.expand([- ("before_the_deadline", time(hour=9, minute=59), False, False, 0),- ("published_day", time(hour=10), True, False, 0),- ("missing_day", time(hour=10), False, False, 1),- ("missing_day_slack_down", time(hour=10), False, True, 2),+ ("before_the_deadline", time(hour=9, minute=59), [1], False, 0, None),+ ("every_day_published", time(hour=10), [], False, 0, None),+ ("missing_yesterday", time(hour=10), [1], False, 1, "2026-10-07"),+ ("missing_older_day", time(hour=10), [4], False, 1, "2026-10-04"),+ ("missing_day_slack_down", time(hour=10), [1], True, 2, "2026-10-07"),])@override_settings(CLOUD_DEPLOYMENT="US", DAGSTER_DOMAIN="dagster.example.com")-def test_overdue_sensor_alerts_once_when_yesterday_is_not_published(- _name: str, check_time: time, published: bool, slack_down: bool, expected_posts: int+def test_overdue_sensor_alerts_once_a_day_about_unpublished_days(+ _name: str,+ check_time: time,+ unpublished_days_ago: list[int],+ slack_down: bool,+ expected_posts: int,+ expected_day: str | None,) -> None:- today = date(2026, 10, 8)instance = dagster.DagsterInstance.ephemeral()- if published:- instance.create_run_for_job(- job_def=warehouse_object_reads_daily_job,- status=dagster.DagsterRunStatus.SUCCESS,- tags={"dagster/partition": (today - timedelta(days=1)).isoformat()},- )+ publish_days_except(instance, unpublished_days_ago)slack = MagicMock()if slack_down:slack.get_client.return_value.chat_postMessage.side_effect = ConnectionError- with time_machine.travel(datetime.combine(today, check_time, tzinfo=UTC), tick=False):+ with time_machine.travel(datetime.combine(CHECK_DAY, check_time, tzinfo=UTC), tick=False):context = dagster.build_sensor_context(instance=instance, resources={"slack": slack})warehouse_object_reads_daily_overdue_sensor(context)warehouse_object_reads_daily_overdue_sensor(@@ -300,4 +334,4 @@ def test_overdue_sensor_alerts_once_when_yesterday_is_not_published(assert len(posts) == expected_postsif expected_posts:assert posts[0].kwargs["channel"] == "#alerts-data-modeling"- assert "2026-10-07" in posts[0].kwargs["text"]+ assert expected_day in posts[0].kwargs["text"]diff --git a/posthog/dags/warehouse_object_reads_daily.py b/posthog/dags/warehouse_object_reads_daily.pyindex 518b6ceae7a1..8bf662f0608e 100644--- a/posthog/dags/warehouse_object_reads_daily.py+++ b/posthog/dags/warehouse_object_reads_daily.py@@ -39,7 +39,8 @@ SPILL_GROUP_BY_AFTER_BYTES = 2 * ONE_GIBMAX_THREADS = 8ROLLUP_NODE_ROLE = NodeRole.AUXROLLUP_START_DATE = "2026-09-17"-SCHEDULE_HOUR_UTC = 7+ATTEMPT_HOURS_UTC = (5, 6, 7)+CATCH_UP_DAYS = 7OVERDUE_AFTER_UTC = time(hour=10)OVERDUE_CHECK_INTERVAL_SECONDS = 60 * 60CONCURRENCY_TAG = {"warehouse_object_reads_backfill_concurrency": "warehouse_object_reads_v1"}@@ -320,14 +321,6 @@ def warehouse_object_reads_daily_job() -> None:rollup_warehouse_object_reads_for_day()-warehouse_object_reads_daily_schedule = dagster.build_schedule_from_partitioned_job(- warehouse_object_reads_daily_job,- hour_of_day=SCHEDULE_HOUR_UTC,- minute_of_hour=0,- default_status=dagster.DefaultScheduleStatus.RUNNING,-)--def has_published_day(instance: dagster.DagsterInstance, day: date) -> bool:return bool(instance.get_run_records(@@ -341,6 +334,36 @@ def has_published_day(instance: dagster.DagsterInstance, day: date) -> bool:)+def unpublished_days(instance: dagster.DagsterInstance, today: date) -> list[date]:+ first_day = max(date.fromisoformat(ROLLUP_START_DATE), today - timedelta(days=CATCH_UP_DAYS))+ days = (first_day + timedelta(days=offset) for offset in range((today - first_day).days))+ return [day for day in days if not has_published_day(instance, day)]+++ job=warehouse_object_reads_daily_job,+ name=f"{warehouse_object_reads_daily_job.name}_schedule",+ cron_schedule=f"0 {','.join(str(hour) for hour in ATTEMPT_HOURS_UTC)} * * *",+ execution_timezone="UTC",+ default_status=dagster.DefaultScheduleStatus.RUNNING,+)+def warehouse_object_reads_daily_schedule(+ context: dagster.ScheduleEvaluationContext,+) -> dagster.RunRequest | dagster.SkipReason:+ active_runs = describe_runs(+ context.instance,+ (warehouse_object_reads_daily_job.name,),+ created_after=datetime.now(UTC) - STUCK_RUN_AGE,+ exclude_run_id="",+ )+ if active_runs:+ return dagster.SkipReason(f"{'; '.join(active_runs)} is queued or executing.")+ missing_days = unpublished_days(context.instance, datetime.now(UTC).date())+ if not missing_days:+ return dagster.SkipReason(f"Every day of the last {CATCH_UP_DAYS} days is published.")+ return dagster.RunRequest(partition_key=missing_days[-1].isoformat())++@dagster.sensor(minimum_interval_seconds=OVERDUE_CHECK_INTERVAL_SECONDS,default_status=dagster.DefaultSensorStatus.RUNNING,@@ -351,22 +374,26 @@ def warehouse_object_reads_daily_overdue_sensor(now = datetime.now(UTC)if now.time() < OVERDUE_AFTER_UTC:return dagster.SkipReason(f"The rollup is not overdue before {OVERDUE_AFTER_UTC} UTC.")- day = now.date() - timedelta(days=1)- if context.cursor == day.isoformat():- return dagster.SkipReason(f"Already alerted that {day} is missing.")- if has_published_day(context.instance, day):- return dagster.SkipReason(f"{day} is published.")+ today = now.date()+ if context.cursor == today.isoformat():+ return dagster.SkipReason(f"Already alerted about missing days on {today}.")+ missing_days = unpublished_days(context.instance, today)+ if not missing_days:+ return dagster.SkipReason(f"Every day of the last {CATCH_UP_DAYS} days is published.")+ missing = ", ".join(day.isoformat() for day in missing_days)if not settings.CLOUD_DEPLOYMENT:- return dagster.SkipReason(f"{day} is missing, but alerts only go out in cloud deployments.")+ return dagster.SkipReason(f"{missing} is missing, but alerts only go out in cloud deployments.")+ attempt_times = ", ".join(f"{hour:02d}:00" for hour in ATTEMPT_HOURS_UTC)job_url = f"https://{settings.DAGSTER_DOMAIN}/locations/clickhouse/jobs/{warehouse_object_reads_daily_job.name}"message = (- f":hourglass: `{warehouse_object_reads_daily_job.name}` has no successful run for {day} "- f"by {OVERDUE_AFTER_UTC:%H:%M} UTC. Check that its schedule is running and that no run is stuck in the queue: "+ f":hourglass: `{warehouse_object_reads_daily_job.name}` has no successful run for {missing} "+ f"by {OVERDUE_AFTER_UTC:%H:%M} UTC. The schedule retries missing days at {attempt_times} UTC "+ f"until they are {CATCH_UP_DAYS} days old. Check that its schedule is running and that no run is stuck in the queue: "f"<{job_url}|View in Dagster>\nEnvironment: {settings.CLOUD_DEPLOYMENT}")blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}]channel = notification_channel_per_team[JobOwners.TEAM_DATA_MODELING.value]if send_slack_alert(context, slack.get_client(), channel, blocks, message):- context.update_cursor(day.isoformat())+ context.update_cursor(today.isoformat())return Nonediff --git a/products/warehouse_suggestions/backend/logic/accept.py b/products/warehouse_suggestions/backend/logic/accept.pyindex 52cd1df1bbd8..10c986066dd8 100644--- a/products/warehouse_suggestions/backend/logic/accept.py+++ b/products/warehouse_suggestions/backend/logic/accept.py@@ -167,7 +167,7 @@ class DeprecateAcceptor(CertificationAcceptor):) -> CreatedAsset:if not request.options.delete_view:return super().accept(suggestion, payload, request)- if suggestion.evidence.get("days_with_data", 0) < RULES.deprecate.min_days_with_data_to_delete:+ if suggestion.evidence.get("days_with_data", 0) < RULES.window_days:raise AcceptOptionRefusedError(DELETE_VIEW_FIELD, DELETE_NEEDS_EVERY_DAY_OF_READS)blocker = _delete_blocker(suggestion.team_id, suggestion.subject_id)if blocker is not None:diff --git a/products/warehouse_suggestions/backend/logic/candidates/deprecate.py b/products/warehouse_suggestions/backend/logic/candidates/deprecate.pyindex d0b2212a36b7..028c16429766 100644--- a/products/warehouse_suggestions/backend/logic/candidates/deprecate.py+++ b/products/warehouse_suggestions/backend/logic/candidates/deprecate.py@@ -26,12 +26,12 @@ class DeprecateCandidate(Candidate):def evaluate(self, context: CandidateContext) -> CandidateResult:rules = context.rules.deprecate- if context.reads.days_with_data < rules.min_days_with_data:+ if context.reads.days_with_data < context.rules.window_days:return CandidateResult(drafts=(),rejections=(),skipped_reason=(- f"needs {rules.min_days_with_data} days of read data, has {context.reads.days_with_data}"+ f"needs {context.rules.window_days} days of read data, has {context.reads.days_with_data}"),)unread = [diff --git a/products/warehouse_suggestions/backend/logic/rules.py b/products/warehouse_suggestions/backend/logic/rules.pyindex 40a87f49e491..029927cfc7a4 100644--- a/products/warehouse_suggestions/backend/logic/rules.py+++ b/products/warehouse_suggestions/backend/logic/rules.py@@ -87,8 +87,6 @@ class CertifyRules:@frozenclass DeprecateRules:- min_days_with_data: int- min_days_with_data_to_delete: intseconds_floor: floatbytes_floor: floatcounts_background_reads: bool@@ -212,8 +210,6 @@ RULES = Rules(top_share=0.10, min_users=5, min_users_floor=2, min_user_share=0.5, min_days=20, min_surfaces=2),deprecate=DeprecateRules(- min_days_with_data=27,- min_days_with_data_to_delete=30,seconds_floor=600.0,bytes_floor=float(TERABYTE),counts_background_reads=True,diff --git a/products/warehouse_suggestions/backend/tests/test_candidates.py b/products/warehouse_suggestions/backend/tests/test_candidates.pyindex ef15e0f5ae29..920b84d23975 100644--- a/products/warehouse_suggestions/backend/tests/test_candidates.py+++ b/products/warehouse_suggestions/backend/tests/test_candidates.py@@ -349,11 +349,11 @@ class TestMaterializeRefreshMode(BaseTest):class TestDeprecateCandidate(BaseTest):@parameterized.expand([- ("a_few_missing_days", 29, [VIEW_ID], None),- ("too_many_missing_days", 26, [], "needs 27 days of read data, has 26"),+ ("every_day", 30, [VIEW_ID], None),+ ("one_missing_day", 29, [], "needs 30 days of read data, has 29"),])- def test_waits_for_most_of_a_window_of_read_data(+ def test_waits_for_every_day_of_read_data(self, _name: str, days_with_data: int, expected_subjects: list, expected_skip: str | None) -> None:candidate_context = context(