Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-modeling): stop sweeping v1 schedules (#118723)

Thiago Salvatore0f37025dd371
diff --git a/posthog/api/test/test_team_deletion.py b/posthog/api/test/test_team_deletion.py
index 0df46fbb7437..1d79a8d36910 100644
--- a/posthog/api/test/test_team_deletion.py
+++ b/posthog/api/test/test_team_deletion.py
@@ -1,9 +1,8 @@
import json
-from datetime import timedelta
import time_machine
from posthog.test.base import NonAtomicBaseTest
-from unittest.mock import ANY, MagicMock, patch
+from unittest.mock import ANY, patch
from django.conf import settings
@@ -278,30 +277,6 @@ class TestTeamDeletionSideEffects(NonAtomicBaseTest):
response = self.client.delete(f"/api/environments/{team.id}")
assert response.status_code == 204
- @patch("posthog.temporal.common.client.sync_connect")
- @patch("products.data_modeling.backend.logic.schedule_reconcile.delete_schedule")
- @patch("products.data_modeling.backend.logic.schedule_reconcile.sync_connect")
- def test_delete_data_modeling_schedules(self, mock_sync_connect, mock_delete_schedule, _batch_export_connect):
- from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
-
- team: Team = Team.objects.create_with_data(initiating_user=self.user, organization=self.organization)
-
- saved_query = DataWarehouseSavedQuery.objects.create(
- team=team,
- name="test_scheduled_query",
- query={"kind": "HogQLQuery", "query": "SELECT 1"},
- sync_frequency_interval=timedelta(hours=1),
- )
-
- mock_temporal = MagicMock()
- mock_sync_connect.return_value = mock_temporal
-
- with execute_deletion_workflows_inline():
- response = self.client.delete(f"/api/environments/{team.id}")
- assert response.status_code == 204
-
- mock_delete_schedule.assert_called_once_with(mock_temporal, schedule_id=str(saved_query.id))
-
@patch("posthog.temporal.common.client.sync_connect")
@patch("products.data_modeling.backend.facade.api.delete_team_data_modeling_schedules")
def test_team_is_still_deleted_when_the_schedule_teardown_fails(self, mock_delete_schedules, _batch_export_connect):
@@ -314,32 +289,3 @@ class TestTeamDeletionSideEffects(NonAtomicBaseTest):
assert response.status_code == 204
mock_delete_schedules.assert_any_call(team.id)
assert not Team.objects.filter(id=team.id).exists()
-
- @patch("posthog.temporal.common.client.sync_connect")
- @patch("products.data_modeling.backend.logic.schedule_reconcile.delete_schedule")
- @patch("products.data_modeling.backend.logic.schedule_reconcile.sync_connect")
- def test_delete_data_modeling_schedules_handles_not_found(
- self, mock_sync_connect, mock_delete_schedule, _batch_export_connect
- ):
- import temporalio.service
-
- from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
-
- team: Team = Team.objects.create_with_data(initiating_user=self.user, organization=self.organization)
-
- DataWarehouseSavedQuery.objects.create(
- team=team,
- name="test_missing_schedule",
- query={"kind": "HogQLQuery", "query": "SELECT 1"},
- sync_frequency_interval=timedelta(hours=1),
- )
-
- mock_temporal = MagicMock()
- mock_sync_connect.return_value = mock_temporal
- mock_delete_schedule.side_effect = temporalio.service.RPCError(
- "not found", temporalio.service.RPCStatusCode.NOT_FOUND, b""
- )
-
- with execute_deletion_workflows_inline():
- response = self.client.delete(f"/api/environments/{team.id}")
- assert response.status_code == 204
diff --git a/products/customer_analytics/backend/facade/api.py b/products/customer_analytics/backend/facade/api.py
index 6cc46d905283..79f23b5ab01c 100644
--- a/products/customer_analytics/backend/facade/api.py
+++ b/products/customer_analytics/backend/facade/api.py
@@ -1838,19 +1838,9 @@ def _resolve_saved_query(team_id: int, saved_query_id: Any, user_access_control:
def _saved_query_meta(saved_query: Any) -> _ProfileWarehouseMeta:
"""Display metadata for a view-bound source. A view's name is already its HogQL name, so it needs
- no table-name resolution. ``sync_frequency_interval`` is null once the view's DAG runs on a shared
- schedule (the frequency lives on the DAG node), which leaves the next run unknown rather than
- guessable — so both schedule fields stay None."""
- interval = saved_query.sync_frequency_interval
- next_sync_at = (
- saved_query.last_run_at + interval if interval is not None and saved_query.last_run_at is not None else None
- )
- return _ProfileWarehouseMeta(
- table_name=saved_query.name,
- sync_frequency_interval_seconds=interval.total_seconds() if interval is not None else None,
- next_sync_at=next_sync_at,
- saved_query_name=saved_query.name,
- )
+ no table-name resolution. A view's frequency lives on its DAG node, which leaves the next run
+ unknown rather than guessable — so both schedule fields stay None."""
+ return _ProfileWarehouseMeta(table_name=saved_query.name, saved_query_name=saved_query.name)
def _resolve_profile_warehouse_meta(
diff --git a/products/data_modeling/backend/logic/saved_query_dag_sync.py b/products/data_modeling/backend/logic/saved_query_dag_sync.py
index ce70d040a226..706ffb7eac3d 100644
--- a/products/data_modeling/backend/logic/saved_query_dag_sync.py
+++ b/products/data_modeling/backend/logic/saved_query_dag_sync.py
@@ -802,9 +802,5 @@ def promote_view_nodes_to_matview(saved_query: "DataWarehouseSavedQuery") -> int
def promote_dag_view_nodes_to_matview(dag: DAG) -> int:
- """Repair a whole DAG's nodes, for the tier conversion that is about to sweep its v1 schedules.
-
- v1 materializes a saved query whatever its node type, so a stranded node keeps running right up
- until that sweep and only goes dark afterwards.
- """
+ """Repair a whole DAG's nodes, for the tier conversion."""
return _promote_view_nodes(Node.objects.filter(dag=dag))
diff --git a/products/data_modeling/backend/logic/schedule_reconcile.py b/products/data_modeling/backend/logic/schedule_reconcile.py
index b541a4140053..77d001217be0 100644
--- a/products/data_modeling/backend/logic/schedule_reconcile.py
+++ b/products/data_modeling/backend/logic/schedule_reconcile.py
@@ -35,8 +35,8 @@ from temporalio.common import RetryPolicy
from temporalio.service import RPCError, RPCStatusCode
from posthog.exceptions_capture import capture_exception
-from posthog.temporal.common.client import async_connect, sync_connect
-from posthog.temporal.common.schedule import a_create_schedule, a_delete_schedule, a_update_schedule, delete_schedule
+from posthog.temporal.common.client import async_connect
+from posthog.temporal.common.schedule import a_create_schedule, a_delete_schedule, a_update_schedule
from posthog.temporal.common.search_attributes import POSTHOG_DAG_ID_KEY
from products.data_modeling.backend.logic.cohort_scheduling import (
@@ -63,7 +63,6 @@ from products.data_modeling.backend.logic.node_frequency import (
FrequencyGraph,
build_frequency_graph,
persist_seed_targets,
- schedulable_nodes,
seed_targets,
set_declared_anchor,
set_declared_target,
@@ -293,11 +292,8 @@ def reconcile_dag_schedules(dag: DAG, *, require_tiered: bool = False, graph: Fr
def convert_dag_to_tiers(dag: DAG, default: timedelta | None = None) -> int:
"""Seed per-node targets from the DAG's current cadence, then reconcile it to per-cadence tier
- schedules. The shared conversion step both entry points (the v1 migrate command and
- reconcile_freshness_schedules) run before clearing the now-redundant saved-query intervals.
- Also repairs nodes a table backs but the graph still calls ephemeral views: v1 runs them
- whatever their type, so they only go dark once the conversion's sweep removes their v1
- schedule.
+ schedules. The shared conversion step of batch_reconcile_teams and reconcile_freshness_schedules.
+ Also repairs nodes a table backs but the graph still calls ephemeral views.
Returns how many targets were seeded.
"""
@@ -311,58 +307,6 @@ def convert_dag_to_tiers(dag: DAG, default: timedelta | None = None) -> int:
return seeded
-def null_saved_query_intervals(dag: DAG, *, only_saved_query_ids: Iterable[str] | None = None) -> int:
- """Clear `sync_frequency_interval` on the DAG's schedulable saved queries — on tiered teams the
- node target is authoritative, and a lingering interval could revive a v1 schedule. Pass
- `only_saved_query_ids` to skip queries whose v1 delete failed, so a re-run retries them.
- """
- from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-
- saved_query_ids = list(schedulable_nodes(dag).values_list("saved_query_id", flat=True))
- if only_saved_query_ids is not None:
- allowed = {str(sq_id) for sq_id in only_saved_query_ids}
- saved_query_ids = [sq_id for sq_id in saved_query_ids if str(sq_id) in allowed]
- return DataWarehouseSavedQuery.objects.filter(
- id__in=saved_query_ids, team_id=dag.team_id, sync_frequency_interval__isnull=False
- ).update(sync_frequency_interval=None)
-
-
-def delete_v1_saved_query_schedules(
- nodes: Iterable[Node], *, team_id: int, dag_id: str, temporal: Client | None = None
-) -> set[str]:
- """Delete live v1 per-query Temporal schedules for the given nodes' saved queries.
-
- NOT_FOUND is treated as already-gone, so this is a safe no-op on a team that never had v1
- schedules — which lets the tiered conversion run on a v1 team directly instead of stacking
- tiers on top of live v1 schedules. Returns the saved-query ids whose delete genuinely failed,
- so the caller keeps their interval for a retry instead of orphaning a live schedule.
- """
- node_list = list(nodes)
- if temporal is None:
- temporal = sync_connect()
- deleted = 0
- failed: list[str] = []
- for node in node_list:
- saved_query = node.saved_query
- if saved_query is None:
- continue
- try:
- delete_schedule(temporal, schedule_id=str(saved_query.id))
- deleted += 1
- except RPCError as e:
- if e.status == RPCStatusCode.NOT_FOUND:
- logger.warning("v1 schedule already gone", saved_query_id=str(saved_query.id), team_id=team_id)
- else:
- failed.append(str(saved_query.id))
- logger.exception("Failed to delete v1 schedule", saved_query_id=str(saved_query.id), team_id=team_id)
- if failed:
- logger.warning(
- "Some v1 schedules could not be deleted", dag_id=dag_id, team_id=team_id, failed_schedule_ids=failed
- )
- logger.info("Swept v1 per-query schedules", dag_id=dag_id, team_id=team_id, deleted=deleted, total=len(node_list))
- return set(failed)
-
-
@dataclasses.dataclass
class DagSchedulePreview:
"""What reconcile would do for a DAG, computed read-only (no schedule writes)."""
@@ -461,37 +405,14 @@ class TeamScheduleTeardownError(Exception):
def delete_team_data_modeling_schedules(team_id: int) -> None:
"""Tear down every Temporal schedule data modeling owns for a team, before CASCADE removes the
- rows that name them. A team runs per-saved-query schedules or per-DAG ones, and converting
- between the two nulls `sync_frequency_interval`, so no field can decide which half to sweep —
- both are swept unconditionally.
+ rows that name them.
Raises `TeamScheduleTeardownError` if any delete failed, so the caller can retry. What outlives
the retries is left to the orphan sweeps, which start from Temporal and so can still find a
schedule after its rows are gone.
"""
- from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-
failed = 0
- saved_query_ids = list(
- DataWarehouseSavedQuery.objects.filter(team_id=team_id).exclude(deleted=True).values_list("id", flat=True)
- )
- if saved_query_ids:
- temporal = sync_connect()
- for saved_query_id in saved_query_ids:
- try:
- delete_schedule(temporal, schedule_id=str(saved_query_id))
- except RPCError as error:
- if error.status == RPCStatusCode.NOT_FOUND:
- continue
- failed += 1
- logger.exception(
- "Failed to delete a saved query schedule for a deleted team",
- saved_query_id=str(saved_query_id),
- team_id=team_id,
- )
- capture_exception(error)
-
for dag_id in DAG.objects.filter(team_id=team_id).values_list("id", flat=True):
if not delete_dag_schedules(str(dag_id)).ok:
failed += 1
diff --git a/products/data_modeling/backend/management/commands/batch_reconcile_teams.py b/products/data_modeling/backend/management/commands/batch_reconcile_teams.py
index 85085b059997..e75ddded8386 100644
--- a/products/data_modeling/backend/management/commands/batch_reconcile_teams.py
+++ b/products/data_modeling/backend/management/commands/batch_reconcile_teams.py
@@ -15,13 +15,7 @@ from posthog.temporal.common.client import async_connect
from posthog.temporal.common.schedule import a_schedule_exists
from products.data_modeling.backend.logic.cohort_scheduling import Tier, tier_schedule_id
-from products.data_modeling.backend.logic.node_frequency import schedulable_nodes
-from products.data_modeling.backend.logic.schedule_reconcile import (
- convert_dag_to_tiers,
- delete_v1_saved_query_schedules,
- null_saved_query_intervals,
- preview_dag_schedules,
-)
+from products.data_modeling.backend.logic.schedule_reconcile import convert_dag_to_tiers, preview_dag_schedules
from products.data_modeling.backend.models.dag import DAG
logger = structlog.get_logger(__name__)
@@ -187,21 +181,8 @@ class Command(BaseCommand):
if not apply:
return
- # Seed every DAG before sweeping any intervals: a query in two DAGs seeds from the shared
- # interval, so sweeping/nulling mid-loop would corrupt the other DAG's seed.
- for dag, _preview, dag_record in plans:
- dag_record["seeded"] = convert_dag_to_tiers(dag)
- for dag, _preview, dag_record in plans:
- nodes = list(schedulable_nodes(dag).select_related("saved_query"))
- sq_ids = [str(node.saved_query_id) for node in nodes if node.saved_query_id is not None]
- failed = delete_v1_saved_query_schedules(nodes, team_id=dag.team_id, dag_id=str(dag.id))
- swept = [sq_id for sq_id in sq_ids if sq_id not in failed]
- dag_record["v1_swept"] = len(swept)
- dag_record["intervals_cleared"] = null_saved_query_intervals(dag, only_saved_query_ids=swept)
- if failed:
- raise Anomaly(f"DAG {dag.name}: {len(failed)} v1 schedule(s) failed to delete")
-
for dag, preview, dag_record in plans:
+ dag_record["seeded"] = convert_dag_to_tiers(dag)
expected = sorted(
tier_schedule_id(str(dag.id), tier.interval, tier.anchor_minutes) for tier in preview.desired_tiers
)
diff --git a/products/data_modeling/backend/management/commands/reconcile_freshness_schedules.py b/products/data_modeling/backend/management/commands/reconcile_freshness_schedules.py
index d8aa063b66ac..eeafa424bf3c 100644
--- a/products/data_modeling/backend/management/commands/reconcile_freshness_schedules.py
+++ b/products/data_modeling/backend/management/commands/reconcile_freshness_schedules.py
@@ -8,12 +8,7 @@ import structlog
from posthog.models.team import Team
-from products.data_modeling.backend.logic.node_frequency import schedulable_nodes
-from products.data_modeling.backend.logic.schedule_reconcile import (
- convert_dag_to_tiers,
- delete_v1_saved_query_schedules,
- null_saved_query_intervals,
-)
+from products.data_modeling.backend.logic.schedule_reconcile import convert_dag_to_tiers
from products.data_modeling.backend.models.dag import DAG
logger = structlog.get_logger(__name__)
@@ -24,9 +19,7 @@ class Command(BaseCommand):
"Convert a team's DAGs to per-cadence-tier schedules: persist a freshness target on every "
"schedulable node lacking one (seeded from its current cadence, never overwriting), then "
"reconcile Temporal so one execute-dag schedule exists per tier — sweeping the legacy "
- "single per-DAG schedule and any v1 per-query schedules, so it is safe on a v1 team "
- "directly. Saved-query sync_frequency_intervals are nulled once targets "
- "are persisted (the node target becomes the only store of frequency intent). Caveat: "
+ "single per-DAG schedule. Caveat: "
"re-running the conversion re-seeds a node whose target was explicitly cleared ('never') "
"from the DAG's interval."
)
@@ -79,18 +72,6 @@ class Command(BaseCommand):
if options["default_interval_seconds"] is not None
else None
)
- # Seed every DAG before touching intervals: a query in two DAGs seeds from the shared
- # interval, so sweeping/nulling mid-loop would corrupt the other DAG's seed.
- seeded_by_dag = [(dag, convert_dag_to_tiers(dag, default=default)) for dag in dag_list]
- for dag, seeded in seeded_by_dag:
- nodes = list(schedulable_nodes(dag).select_related("saved_query"))
- sq_ids = [str(node.saved_query_id) for node in nodes if node.saved_query_id is not None]
- # Sweep any legacy v1 per-query schedules (no-op on a v2 team); null only the intervals
- # whose v1 schedule is gone, so a failed delete keeps its interval for a re-run.
- failed = delete_v1_saved_query_schedules(nodes, team_id=dag.team_id, dag_id=str(dag.id))
- swept = [sq_id for sq_id in sq_ids if sq_id not in failed]
- cleared = null_saved_query_intervals(dag, only_saved_query_ids=swept)
- self.stdout.write(
- f"DAG {dag.name} ({dag.id}): seeded {seeded} target(s), reconciled, "
- f"swept {len(swept)} v1 schedule(s), cleared {cleared} interval(s)"
- )
+ for dag in dag_list:
+ seeded = convert_dag_to_tiers(dag, default=default)
+ self.stdout.write(f"DAG {dag.name} ({dag.id}): seeded {seeded} target(s), reconciled")
diff --git a/products/data_modeling/backend/models/datawarehouse_saved_query.py b/products/data_modeling/backend/models/datawarehouse_saved_query.py
index c786b5f72701..88678d22587a 100644
--- a/products/data_modeling/backend/models/datawarehouse_saved_query.py
+++ b/products/data_modeling/backend/models/datawarehouse_saved_query.py
@@ -330,7 +330,7 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
# schedule does not count: reconcile refuses to add tiers beside one, so bootstrap sweeps it.
# This Temporal lookup stays inside the try so that, if it fails, we honor the failure
# contract below rather than leaving is_materialized=True with no schedule backing it.
- on_v2 = self.id in get_v2_saved_query_ids([self.id], team_id=self.team_id, tiered_only=True)
+ on_v2 = self.id in get_v2_saved_query_ids([self.id], tiered_only=True)
node = (
Node.objects.filter(team_id=self.team_id, saved_query_id=self.id)
.select_related("dag", "dag__team")
diff --git a/products/data_modeling/backend/schedule.py b/products/data_modeling/backend/schedule.py
index 56fe0c1e5bc8..240603a7e4c3 100644
--- a/products/data_modeling/backend/schedule.py
+++ b/products/data_modeling/backend/schedule.py
@@ -35,7 +35,6 @@ from posthog.temporal.common.search_attributes import (
)
from products.data_modeling.backend.logic.cohort_scheduling import dag_id_from_schedule_id, is_tier_schedule_id
-from products.data_modeling.backend.models.dag import DAG
from products.data_modeling.backend.models.node import Node
# A schedule id is the bare DAG id, or "{dag_id}:{interval_seconds}" for a per-cadence-tier
@@ -60,10 +59,6 @@ async def get_v2_scheduled_dag_ids(
) -> set[str]:
"""Return the IDs of DAGs that already have a v2 `data-modeling-execute-dag` Temporal schedule.
- A DAG appearing here has been migrated off the frozen v1 backend. Callers performing v1
- schedule operations must skip these DAGs' saved queries so they never re-create or revive a
- v1 schedule for a DAG already running on v2.
-
`tiered_only` counts only cadence-tier schedules (`{dag_id}:{seconds}`), ignoring the bare
whole-DAG `{dag_id}` schedule the legacy migration left behind. A bare schedule fires the DAG
but honours no frequency target, so a caller about to apply one must treat such a DAG as not
@@ -102,25 +97,13 @@ async def get_v2_scheduled_dag_ids(
def get_v2_saved_query_ids(
- candidate_ids: Collection[uuid.UUID] | None = None, *, team_id: int | None = None, tiered_only: bool = False
+ candidate_ids: Collection[uuid.UUID] | None = None, *, tiered_only: bool = False
) -> set[uuid.UUID]:
"""Return saved query IDs whose DAG already runs on a v2 schedule.
`tiered_only` narrows "on v2" to DAGs with a cadence tier, as on `get_v2_scheduled_dag_ids`.
- A saved query counts as on v2 when any DAG it belongs to has a v2 schedule, because it can sit
- in several DAGs and one v1-scheduled placement does not make a v1 schedule safe to mint.
-
- `team_id` extends that to saved queries with no node, answering from the team's DAGs instead.
- A node can be absent because `sync_saved_query_to_dag` rolls its creation back when dependency
- resolution raises, and reading "no node" as "not on v2" mints a v1 per-query schedule beside the team's
- live tier, which then materializes the query twice on every cycle. Only a caller that is about
- to create a v1 schedule needs this, so it stays opt-in: it answers about the team rather than
- about a placement, and such a caller must still check for a node before scheduling. Pass the
- team the candidates belong to, never one derived from the ids themselves.
-
- Optionally restrict the lookup to `candidate_ids` to keep the query bounded. These saved
- queries must be skipped by v1 schedule commands so we never undo migration progress.
+ Optionally restrict the lookup to `candidate_ids` to keep the query bounded.
"""
if candidate_ids is not None:
if not candidate_ids:
@@ -134,15 +117,6 @@ def get_v2_saved_query_ids(
if dag_id is not None:
dag_ids_by_saved_query[saved_query_id].add(str(dag_id))
- if team_id is not None:
- nodeless = set(candidate_ids) - dag_ids_by_saved_query.keys()
- team_dag_ids = (
- {str(dag_id) for dag_id in DAG.objects.filter(team_id=team_id).values_list("id", flat=True)}
- if nodeless
- else set()
- )
- if team_dag_ids:
- dag_ids_by_saved_query.update(dict.fromkeys(nodeless, team_dag_ids))
if not dag_ids_by_saved_query:
return set()
diff --git a/products/data_modeling/backend/test/test_batch_reconcile_teams.py b/products/data_modeling/backend/test/test_batch_reconcile_teams.py
index 6522ca7ff678..ff4488a6b91c 100644
--- a/products/data_modeling/backend/test/test_batch_reconcile_teams.py
+++ b/products/data_modeling/backend/test/test_batch_reconcile_teams.py
@@ -42,8 +42,6 @@ class TestBatchReconcileTeams(BaseTest):
def _run(self, team_ids: str, *extra_args, existing=(), verify_listing=None):
out = StringIO()
with (
- mock.patch(f"{RECONCILE}.sync_connect"),
- mock.patch(f"{RECONCILE}.delete_schedule"),
mock.patch(
f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=_temporal_listing(list(existing)))
),
@@ -95,9 +93,6 @@ class TestBatchReconcileTeams(BaseTest):
assert team_record["dags"][0]["verified"] is True
create.assert_called_once()
assert create.call_args.kwargs["id"] == expected_tier
- assert node.saved_query is not None
- node.saved_query.refresh_from_db()
- assert node.saved_query.sync_frequency_interval is None
def test_verify_mismatch_halts(self):
# if the live schedule set after apply is not exactly the planned tier set, the batch
@@ -147,8 +142,6 @@ class TestBatchReconcileTeams(BaseTest):
out = StringIO()
with (
mock.patch(f"{BATCH}.time.sleep") as sleep,
- mock.patch(f"{RECONCILE}.sync_connect"),
- mock.patch(f"{RECONCILE}.delete_schedule"),
mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(side_effect=[rate_limit, listing])),
mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()),
mock.patch(f"{RECONCILE}.a_delete_schedule", new=mock.AsyncMock()),
diff --git a/products/data_modeling/backend/test/test_reconcile_freshness_schedules.py b/products/data_modeling/backend/test/test_reconcile_freshness_schedules.py
index 75482daa92c7..21907fb399e2 100644
--- a/products/data_modeling/backend/test/test_reconcile_freshness_schedules.py
+++ b/products/data_modeling/backend/test/test_reconcile_freshness_schedules.py
@@ -39,8 +39,6 @@ class TestReconcileFreshnessSchedules(BaseTest):
legacy_id = str(dag.id)
with (
- mock.patch(f"{RECONCILE}.sync_connect"),
- mock.patch(f"{RECONCILE}.delete_schedule"),
mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=_temporal_listing([legacy_id]))),
mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()) as create,
mock.patch(f"{RECONCILE}.a_delete_schedule", new=mock.AsyncMock()) as delete,
@@ -49,31 +47,10 @@ class TestReconcileFreshnessSchedules(BaseTest):
node.refresh_from_db()
self.assertEqual(get_declared_target(node), H6)
- assert node.saved_query is not None
- node.saved_query.refresh_from_db()
- self.assertIsNone(node.saved_query.sync_frequency_interval)
create.assert_called_once()
self.assertEqual(create.call_args.kwargs["id"], tier_schedule_id(legacy_id, H6))
delete.assert_called_once_with(mock.ANY, schedule_id=legacy_id)
- def test_sweeps_live_v1_schedules(self):
- dag, node = self._legacy_dag()
- assert node.saved_query is not None
- v1_id = str(node.saved_query.id)
-
- with (
- mock.patch(f"{RECONCILE}.sync_connect"),
- mock.patch(f"{RECONCILE}.delete_schedule") as delete_v1,
- mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=_temporal_listing([]))),
- mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()),
- mock.patch(f"{RECONCILE}.a_delete_schedule", new=mock.AsyncMock()),
- ):
- call_command("reconcile_freshness_schedules", "--team-id", str(self.team.pk), stdout=StringIO())
-
- delete_v1.assert_called_once_with(mock.ANY, schedule_id=v1_id)
- node.saved_query.refresh_from_db()
- self.assertIsNone(node.saved_query.sync_frequency_interval)
-
def test_dry_run_rejects_default_interval(self):
self._legacy_dag()
with self.assertRaisesRegex(CommandError, "default-interval-seconds"):
@@ -87,33 +64,6 @@ class TestReconcileFreshnessSchedules(BaseTest):
stdout=StringIO(),
)
- def test_query_in_two_dags_seeds_from_its_own_interval(self):
- D1 = timedelta(days=1)
- saved_query = DataWarehouseSavedQuery.objects.create(
- name="shared",
- team=self.team,
- query={"query": "SELECT 1", "kind": "HogQLQuery"},
- sync_frequency_interval=H6,
- )
- dag_a = DAG.objects.create(team=self.team, name="A", sync_frequency_interval=D1)
- dag_b = DAG.objects.create(team=self.team, name="B", sync_frequency_interval=D1)
- node_a = Node.objects.create(team=self.team, dag=dag_a, saved_query=saved_query, type=NodeType.VIEW)
- node_b = Node.objects.create(team=self.team, dag=dag_b, saved_query=saved_query, type=NodeType.VIEW)
-
- with (
- mock.patch(f"{RECONCILE}.sync_connect"),
- mock.patch(f"{RECONCILE}.delete_schedule"),
- mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=_temporal_listing([]))),
- mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()),
- mock.patch(f"{RECONCILE}.a_delete_schedule", new=mock.AsyncMock()),
- ):
- call_command("reconcile_freshness_schedules", "--team-id", str(self.team.pk), stdout=StringIO())
-
- node_a.refresh_from_db()
- node_b.refresh_from_db()
- self.assertEqual(get_declared_target(node_a), H6)
- self.assertEqual(get_declared_target(node_b), H6)
-
def test_dry_run_delegates_to_preview_and_writes_nothing(self):
dag, node = self._legacy_dag()
diff --git a/products/data_modeling/backend/test/test_schedule.py b/products/data_modeling/backend/test/test_schedule.py
index e6dd438d10a4..3c9b8809992a 100644
--- a/products/data_modeling/backend/test/test_schedule.py
+++ b/products/data_modeling/backend/test/test_schedule.py
@@ -336,24 +336,6 @@ class TestV2ScheduleGuard(BaseTest):
query={"query": "SELECT 1", "kind": "HogQLQuery"},
)
- def test_saved_query_without_a_node_is_not_reported_as_v1_eligible(self):
- nodeless = self._saved_query("sync_failed")
- with mock.patch(
- "products.data_modeling.backend.schedule.get_v2_scheduled_dag_ids",
- return_value={str(self.v2_dag.id)},
- ):
- result = get_v2_saved_query_ids([nodeless.id], team_id=self.team.pk)
- assert result == {nodeless.id}
-
- def test_saved_query_without_a_node_stays_v1_eligible_when_no_dag_is_v2_scheduled(self):
- nodeless = self._saved_query("sync_failed_on_v1_team")
- with mock.patch(
- "products.data_modeling.backend.schedule.get_v2_scheduled_dag_ids",
- return_value=set(),
- ):
- result = get_v2_saved_query_ids([nodeless.id], team_id=self.team.pk)
- assert result == set()
-
@parameterized.expand([("v2_node_created_first", True), ("v1_node_created_first", False)])
def test_saved_query_in_two_dags_is_on_v2_when_either_dag_is(self, _name: str, v2_first: bool):
shared = self._saved_query("in_two_dags")
@@ -363,7 +345,7 @@ class TestV2ScheduleGuard(BaseTest):
"products.data_modeling.backend.schedule.get_v2_scheduled_dag_ids",
return_value={str(self.v2_dag.id)},
):
- result = get_v2_saved_query_ids([shared.id], team_id=self.team.pk)
+ result = get_v2_saved_query_ids([shared.id])
assert result == {shared.id}
diff --git a/products/data_modeling/backend/test/test_schedule_materialization.py b/products/data_modeling/backend/test/test_schedule_materialization.py
index 68c4a8ed2505..1a2dd15e0bd7 100644
--- a/products/data_modeling/backend/test/test_schedule_materialization.py
+++ b/products/data_modeling/backend/test/test_schedule_materialization.py
@@ -105,7 +105,6 @@ class TestScheduleMaterializationV2Guard(BaseTest):
node = Node.objects.get(saved_query=self.sq)
with (
mock.patch(GET_V2_DAG_IDS, return_value=set()),
- mock.patch(f"{RECONCILE}.sync_connect"),
mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=_no_schedules())),
mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()) as create,
mock.patch(f"{NODE_MAT}.sync_connect"),
@@ -128,7 +127,6 @@ class TestScheduleMaterializationV2Guard(BaseTest):
bare = temporal_listing([str(self.dag.id)])
with (
mock.patch("products.data_modeling.backend.schedule.async_connect", new=mock.AsyncMock(return_value=bare)),
- mock.patch(f"{RECONCILE}.sync_connect"),
mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=bare)),
mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()) as create,
mock.patch(f"{RECONCILE}.a_delete_schedule", new=mock.AsyncMock()) as delete,
@@ -154,7 +152,6 @@ class TestScheduleMaterializationV2Guard(BaseTest):
self.sq.save(update_fields=["is_materialized"])
with (
mock.patch(GET_V2_DAG_IDS, return_value=set()),
- mock.patch(f"{RECONCILE}.sync_connect"),
mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=_no_schedules())),
mock.patch(
f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock(side_effect=Exception("temporal unavailable"))
@@ -177,7 +174,6 @@ class TestScheduleMaterializationV2Guard(BaseTest):
self.sq.save(update_fields=["sync_frequency_interval"])
with (
mock.patch(GET_V2_DAG_IDS, return_value=set()),
- mock.patch(f"{RECONCILE}.sync_connect"),
mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=_no_schedules())),
mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()) as create,
self.captureOnCommitCallbacks(execute=True),
diff --git a/products/data_modeling/backend/test/test_schedule_reconcile.py b/products/data_modeling/backend/test/test_schedule_reconcile.py
index e3e4a0bbc000..0a74f97a0819 100644
--- a/products/data_modeling/backend/test/test_schedule_reconcile.py
+++ b/products/data_modeling/backend/test/test_schedule_reconcile.py
@@ -542,14 +542,11 @@ class TestPromoteDagViewNodesToMatview(BaseTest):
assert has_run.type == NodeType.MAT_VIEW
def test_conversion_to_tiers_repairs_stranded_nodes(self):
- # The v1 sweep that follows conversion is what makes a view-typed node go dark, so the
- # repair has to happen as part of converting, not on some later pass.
dag = DAG.get_or_create_default(self.team)
stranded = self._backed(dag, "stranded", NodeType.VIEW)
set_declared_target(stranded, M15)
with (
- mock.patch(f"{RECONCILE}.sync_connect", return_value=no_existing_schedules()),
mock.patch(f"{RECONCILE}.async_connect", return_value=no_existing_schedules()),
mock.patch(f"{RECONCILE}.a_create_schedule"),
):
@@ -612,27 +609,15 @@ class TestDeleteDagSchedules(SimpleTestCase):
class TestDeleteTeamDataModelingSchedules(BaseTest):
- def test_sweeps_both_halves_for_a_team_converted_to_dag_schedules(self):
- # Conversion nulls sync_frequency_interval, so no field can decide which half to sweep.
- saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="converted_query",
- query={"kind": "HogQLQuery", "query": "SELECT 1"},
- sync_frequency_interval=None,
- )
+ def test_deletes_the_schedules_of_every_dag_of_the_team(self):
dag = DAG.objects.create(team=self.team, name="Default")
- with (
- mock.patch(f"{RECONCILE}.sync_connect"),
- mock.patch(f"{RECONCILE}.delete_schedule") as delete_query,
- mock.patch(
- f"{RECONCILE}.delete_dag_schedules",
- return_value=DagScheduleTeardown(ok=True, deleted=(str(dag.id),)),
- ) as delete_dag,
- ):
+ with mock.patch(
+ f"{RECONCILE}.delete_dag_schedules",
+ return_value=DagScheduleTeardown(ok=True, deleted=(str(dag.id),)),
+ ) as delete_dag:
delete_team_data_modeling_schedules(self.team.id)
- delete_query.assert_called_once_with(mock.ANY, schedule_id=str(saved_query.id))
delete_dag.assert_called_once_with(str(dag.id))
def test_raises_when_a_dag_teardown_fails_so_the_activity_retries(self):
diff --git a/products/data_modeling/backend/tests/test_managed_viewset_providers.py b/products/data_modeling/backend/tests/test_managed_viewset_providers.py
index 3a9172516a15..59d3f6a1ce63 100644
--- a/products/data_modeling/backend/tests/test_managed_viewset_providers.py
+++ b/products/data_modeling/backend/tests/test_managed_viewset_providers.py
@@ -181,7 +181,6 @@ class TestManagedViewSetProviders(BaseTest):
with (
patch.dict(_expected_views_providers, clear=True),
patch(GET_V2_DAG_IDS, return_value=set()),
- patch(f"{RECONCILE}.sync_connect"),
patch(f"{RECONCILE}.async_connect", new=AsyncMock(return_value=_no_schedules())),
patch(f"{RECONCILE}.a_create_schedule", new=AsyncMock()) as create,
patch(f"{NODE_MAT}.sync_connect"),