Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): create tier schedule when update finds it gone (#116697)

posthog[bot]42a58406e3e1
diff --git a/products/data_modeling/backend/logic/schedule_reconcile.py b/products/data_modeling/backend/logic/schedule_reconcile.py
index d1e91811f312..ce33baf2031f 100644
--- a/products/data_modeling/backend/logic/schedule_reconcile.py
+++ b/products/data_modeling/backend/logic/schedule_reconcile.py
@@ -432,23 +432,31 @@ async def _apply_reconciliation(
# uncovered; on failure, best-effort-delete the tiers we created (already-applied updates
# stay — a re-run converges) without letting a failed delete mask the original error.
created: list[str] = []
+
+ async def create_or_converge(schedule_id: str, schedule: Schedule) -> None:
+ try:
+ await a_create_schedule(temporal, id=schedule_id, schedule=schedule, search_attributes=search_attributes)
+ created.append(schedule_id)
+ except ScheduleAlreadyRunningError:
+ # A concurrent reconcile of the same DAG created this tier from the same graph;
+ # converge onto it. It is deliberately NOT in `created`: it isn't ours to roll back.
+ await a_update_schedule(temporal, id=schedule_id, schedule=schedule, search_attributes=search_attributes)
+
try:
for schedule_id, (tier, node_ids) in plan.to_create.items():
+ await create_or_converge(schedule_id, _build_tier_schedule(dag_id, team_id, team_timezone, tier, node_ids))
+ for schedule_id, (tier, node_ids) in plan.to_update.items():
schedule = _build_tier_schedule(dag_id, team_id, team_timezone, tier, node_ids)
try:
- await a_create_schedule(
- temporal, id=schedule_id, schedule=schedule, search_attributes=search_attributes
- )
- created.append(schedule_id)
- except ScheduleAlreadyRunningError:
- # A concurrent reconcile of the same DAG created this tier from the same graph;
- # converge onto it. It is deliberately NOT in `created`: it isn't ours to roll back.
await a_update_schedule(
temporal, id=schedule_id, schedule=schedule, search_attributes=search_attributes
)
- for schedule_id, (tier, node_ids) in plan.to_update.items():
- schedule = _build_tier_schedule(dag_id, team_id, team_timezone, tier, node_ids)
- await a_update_schedule(temporal, id=schedule_id, schedule=schedule, search_attributes=search_attributes)
+ except RPCError as error:
+ if error.status != RPCStatusCode.NOT_FOUND:
+ raise
+ # The listing reads Temporal's eventually consistent visibility store, so it can
+ # still name a tier that a teardown or a concurrent reconcile just deleted.
+ await create_or_converge(schedule_id, schedule)
except Exception:
for schedule_id in created:
try:
diff --git a/products/data_modeling/backend/test/test_schedule_reconcile.py b/products/data_modeling/backend/test/test_schedule_reconcile.py
index a55da8b02fcd..4f79648e9c94 100644
--- a/products/data_modeling/backend/test/test_schedule_reconcile.py
+++ b/products/data_modeling/backend/test/test_schedule_reconcile.py
@@ -23,6 +23,7 @@ from products.data_modeling.backend.logic.schedule_reconcile import (
DagScheduleTeardown,
TeamScheduleTeardownError,
apply_saved_query_frequency_anchor,
+ bootstrap_dag_to_tiers,
convert_dag_to_tiers,
delete_dag_schedules,
delete_team_data_modeling_schedules,
@@ -328,6 +329,48 @@ class TestReconcileDagSchedules(BaseTest):
self.assertEqual(update.call_args.kwargs["id"], tier_schedule_id(str(dag.id), M15))
delete.assert_not_called()
+ def test_listed_tier_that_is_already_gone_is_recreated_and_stays_materialized(self):
+ # the listing reads Temporal's eventually consistent visibility store, so it can name a
+ # tier that is already deleted; a failed bootstrap would mark the query not materialized
+ dag = DAG.get_or_create_default(self.team)
+ source = _table_node(self.team, dag, "events", {"origin": "posthog"})
+ endpoint = _saved_query_node(self.team, dag, "ep", NodeType.ENDPOINT)
+ Edge.objects.create(team=self.team, dag=dag, source=source, target=endpoint)
+ set_declared_target(endpoint, M15)
+ saved_query = endpoint.saved_query
+ assert saved_query is not None
+ saved_query.is_materialized = True
+ saved_query.save(update_fields=["is_materialized"])
+
+ tier_id = tier_schedule_id(str(dag.id), M15)
+
+ async def fake_list_schedules(*_args, **_kwargs):
+ async def gen():
+ yield _listing(tier_id)
+
+ return gen()
+
+ temporal = mock.Mock()
+ temporal.list_schedules = fake_list_schedules
+
+ with (
+ mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=temporal)),
+ mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()) as create,
+ mock.patch(
+ f"{RECONCILE}.a_update_schedule",
+ new=mock.AsyncMock(side_effect=RPCError("schedule not found", RPCStatusCode.NOT_FOUND, b"")),
+ ),
+ mock.patch(f"{RECONCILE}.a_delete_schedule", new=mock.AsyncMock()) as delete,
+ self.captureOnCommitCallbacks(execute=True),
+ ):
+ bootstrap_dag_to_tiers(dag, requested_by=saved_query)
+
+ create.assert_called_once()
+ self.assertEqual(create.call_args.kwargs["id"], tier_id)
+ delete.assert_not_called()
+ saved_query.refresh_from_db()
+ self.assertTrue(saved_query.is_materialized)
+
def test_refuses_non_bucket_tier_before_touching_temporal(self):
# the guard must fire before any Temporal call — a non-bucket tier would crash
# build_schedule_spec mid-apply and leave the DAG partially reconciled