:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): create tier schedule when update finds it gone (#116697)
posthog[bot]42a58406e3e1
diff --git a/products/data_modeling/backend/logic/schedule_reconcile.py b/products/data_modeling/backend/logic/schedule_reconcile.pyindex d1e91811f312..ce33baf2031f 100644--- a/products/data_modeling/backend/logic/schedule_reconcile.py+++ b/products/data_modeling/backend/logic/schedule_reconcile.py@@ -432,23 +432,31 @@ async def _apply_reconciliation(# uncovered; on failure, best-effort-delete the tiers we created (already-applied updates# stay — a re-run converges) without letting a failed delete mask the original error.created: list[str] = []++ async def create_or_converge(schedule_id: str, schedule: Schedule) -> None:+ try:+ await a_create_schedule(temporal, id=schedule_id, schedule=schedule, search_attributes=search_attributes)+ created.append(schedule_id)+ except ScheduleAlreadyRunningError:+ # A concurrent reconcile of the same DAG created this tier from the same graph;+ # converge onto it. It is deliberately NOT in `created`: it isn't ours to roll back.+ await a_update_schedule(temporal, id=schedule_id, schedule=schedule, search_attributes=search_attributes)+try:for schedule_id, (tier, node_ids) in plan.to_create.items():+ await create_or_converge(schedule_id, _build_tier_schedule(dag_id, team_id, team_timezone, tier, node_ids))+ for schedule_id, (tier, node_ids) in plan.to_update.items():schedule = _build_tier_schedule(dag_id, team_id, team_timezone, tier, node_ids)try:- await a_create_schedule(- temporal, id=schedule_id, schedule=schedule, search_attributes=search_attributes- )- created.append(schedule_id)- except ScheduleAlreadyRunningError:- # A concurrent reconcile of the same DAG created this tier from the same graph;- # converge onto it. It is deliberately NOT in `created`: it isn't ours to roll back.await a_update_schedule(temporal, id=schedule_id, schedule=schedule, search_attributes=search_attributes)- for schedule_id, (tier, node_ids) in plan.to_update.items():- schedule = _build_tier_schedule(dag_id, team_id, team_timezone, tier, node_ids)- await a_update_schedule(temporal, id=schedule_id, schedule=schedule, search_attributes=search_attributes)+ except RPCError as error:+ if error.status != RPCStatusCode.NOT_FOUND:+ raise+ # The listing reads Temporal's eventually consistent visibility store, so it can+ # still name a tier that a teardown or a concurrent reconcile just deleted.+ await create_or_converge(schedule_id, schedule)except Exception:for schedule_id in created:try:diff --git a/products/data_modeling/backend/test/test_schedule_reconcile.py b/products/data_modeling/backend/test/test_schedule_reconcile.pyindex a55da8b02fcd..4f79648e9c94 100644--- a/products/data_modeling/backend/test/test_schedule_reconcile.py+++ b/products/data_modeling/backend/test/test_schedule_reconcile.py@@ -23,6 +23,7 @@ from products.data_modeling.backend.logic.schedule_reconcile import (DagScheduleTeardown,TeamScheduleTeardownError,apply_saved_query_frequency_anchor,+ bootstrap_dag_to_tiers,convert_dag_to_tiers,delete_dag_schedules,delete_team_data_modeling_schedules,@@ -328,6 +329,48 @@ class TestReconcileDagSchedules(BaseTest):self.assertEqual(update.call_args.kwargs["id"], tier_schedule_id(str(dag.id), M15))delete.assert_not_called()+ def test_listed_tier_that_is_already_gone_is_recreated_and_stays_materialized(self):+ # the listing reads Temporal's eventually consistent visibility store, so it can name a+ # tier that is already deleted; a failed bootstrap would mark the query not materialized+ dag = DAG.get_or_create_default(self.team)+ source = _table_node(self.team, dag, "events", {"origin": "posthog"})+ endpoint = _saved_query_node(self.team, dag, "ep", NodeType.ENDPOINT)+ Edge.objects.create(team=self.team, dag=dag, source=source, target=endpoint)+ set_declared_target(endpoint, M15)+ saved_query = endpoint.saved_query+ assert saved_query is not None+ saved_query.is_materialized = True+ saved_query.save(update_fields=["is_materialized"])++ tier_id = tier_schedule_id(str(dag.id), M15)++ async def fake_list_schedules(*_args, **_kwargs):+ async def gen():+ yield _listing(tier_id)++ return gen()++ temporal = mock.Mock()+ temporal.list_schedules = fake_list_schedules++ with (+ mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=temporal)),+ mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()) as create,+ mock.patch(+ f"{RECONCILE}.a_update_schedule",+ new=mock.AsyncMock(side_effect=RPCError("schedule not found", RPCStatusCode.NOT_FOUND, b"")),+ ),+ mock.patch(f"{RECONCILE}.a_delete_schedule", new=mock.AsyncMock()) as delete,+ self.captureOnCommitCallbacks(execute=True),+ ):+ bootstrap_dag_to_tiers(dag, requested_by=saved_query)++ create.assert_called_once()+ self.assertEqual(create.call_args.kwargs["id"], tier_id)+ delete.assert_not_called()+ saved_query.refresh_from_db()+ self.assertTrue(saved_query.is_materialized)+def test_refuses_non_bucket_tier_before_touching_temporal(self):# the guard must fire before any Temporal call — a non-bucket tier would crash# build_schedule_spec mid-apply and leave the DAG partially reconciled