:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(data-modeling): delete the finished dag consolidation commands (#118706)
Thiago Salvatore98e9d0eb9612
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex f7c8100ee990..c2f446532ff1 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -1416,7 +1416,6 @@3 products/batch_exports/backend/temporal/pipeline/internal_stage.py3 products/conversations/backend/temporal/zendesk_import/workflows.py3 products/data_modeling/backend/logic/gitsync/config_parser.py-3 products/data_modeling/backend/management/commands/consolidate_dags.py3 products/data_warehouse/backend/logic/external_data_source/webhooks.py3 products/experiments/stats/shared/statistics.py3 products/marketing_analytics/backend/services/data_source_health.pydiff --git a/products/data_modeling/backend/logic/node_frequency.py b/products/data_modeling/backend/logic/node_frequency.pyindex 359378be030b..8bf6e21d7c03 100644--- a/products/data_modeling/backend/logic/node_frequency.py+++ b/products/data_modeling/backend/logic/node_frequency.py@@ -9,7 +9,7 @@ typed column.import uuidimport dataclasses-from collections.abc import Collection, Iterable+from collections.abc import Iterablefrom datetime import timedeltafrom django.db.models import Q, QuerySet@@ -356,13 +356,11 @@ def schedulable_nodes(dag: DAG) -> QuerySet[Node]:return Node.objects.filter(dag=dag, type__in=SAVED_QUERY_NODE_TYPES).exclude(saved_query__deleted=True)-def persist_seed_targets(- dag: DAG, default: timedelta | None = None, only_node_ids: Collection[str] | None = None-) -> int:+def persist_seed_targets(dag: DAG, default: timedelta | None = None) -> int:"""Persist a seed target on every schedulable node lacking one; never overwrites. Each seed isnormalized (see `normalize_seed_target`) so the persisted target equals what the scheduler willrun. `default` is the operator escape hatch for scheduled DAGs with no interval metadata- anywhere. `only_node_ids` limits the seeding to those nodes. Returns how many targets were written.+ anywhere. Returns how many targets were written."""graph = build_frequency_graph(dag)floors = all_source_floors(graph.edges, graph.source_intervals)@@ -370,8 +368,6 @@ def persist_seed_targets(written = 0for node in schedulable_nodes(dag):node_id = str(node.id)- if only_node_ids is not None and node_id not in only_node_ids:- continueif get_declared_target(node) is not None:continueraw = seeds.get(node_id, default)diff --git a/products/data_modeling/backend/logic/saved_query_dag_sync.py b/products/data_modeling/backend/logic/saved_query_dag_sync.pyindex 593bf689cc13..ce70d040a226 100644--- a/products/data_modeling/backend/logic/saved_query_dag_sync.py+++ b/products/data_modeling/backend/logic/saved_query_dag_sync.py@@ -12,8 +12,6 @@ from posthog.hogql.database.models import SavedQuery as HogQLSavedQueryfrom posthog.hogql.database.s3_table import DataWarehouseTable as HogQLDataWarehouseTablefrom posthog.hogql.errors import QueryError-from posthog.dataclasses import frozen-from products.data_modeling.backend.facade.contracts import Dependentfrom products.data_modeling.backend.facade.system_tables import DATA_MODELING_ALLOWED_SYSTEM_TABLESfrom products.data_modeling.backend.logic.node_suspension import clear_suspension_if_query_changed@@ -810,80 +808,3 @@ def promote_dag_view_nodes_to_matview(dag: DAG) -> int:until that sweep and only goes dark afterwards."""return _promote_view_nodes(Node.objects.filter(dag=dag))---def stale_managed_view_copies() -> QuerySet[Node]:- """Nodes of managed views outside the managed DAG, for views the managed DAG already holds."""- managed_saved_query_ids = Node.objects.filter(- dag__name=REVENUE_ANALYTICS_DAG_NAME, saved_query__isnull=False- ).values("saved_query_id")- return Node.objects.filter(- saved_query__managed_viewset__isnull=False, saved_query_id__in=managed_saved_query_ids- ).exclude(dag__name=REVENUE_ANALYTICS_DAG_NAME)---@frozen-class _Reader:- node_id: UUID- saved_query_id: UUID---def _user_readers_of(copies: QuerySet[Node]) -> list[_Reader]:- rows = (- Node.objects.filter(- incoming_edges__source__in=copies,- saved_query__isnull=False,- saved_query__managed_viewset__isnull=True,- saved_query__deleted=False,- )- .values_list("id", "saved_query_id")- .distinct()- )- return [_Reader(node_id=node_id, saved_query_id=saved_query_id) for node_id, saved_query_id in rows]---def _repoint_reader(team_id: int, reader: _Reader, database: Database) -> None:- from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery-- try:- with transaction.atomic():- saved_query = DataWarehouseSavedQuery.objects.select_for_update(no_key=True).get(- team_id=team_id, id=reader.saved_query_id- )- node = Node.objects.select_related("dag").filter(team_id=team_id, id=reader.node_id).first()- if node is None:- return- sync_saved_query_to_dag(saved_query, dag=node.dag, reconcile=False, database=database)- except Exception:- logger.exception("Could not repoint a reader of a stale managed view copy", node_id=str(reader.node_id))---def _delete_unread_copies(team_id: int, dag: DAG, copies: QuerySet[Node]) -> None:- materializing = DataModelingJob.objects.filter(- team_id=team_id, status=DataModelingJobStatus.RUNNING, saved_query_id__isnull=False- ).values("saved_query_id")- unread = copies.filter(dag=dag, outgoing_edges__isnull=True).exclude(saved_query_id__in=materializing)- deleted_any = False- with transaction.atomic():- lock_dag(team_id, dag.id)- lock_dag_placements(team_id, dag.id)- while unread.exists():- delete_nodes(team_id, unread)- deleted_any = True- if deleted_any:- maybe_reconcile_dag(dag)---def retire_stale_managed_view_copies(team_id: int, copy_ids: Iterable[UUID]) -> None:- copies = stale_managed_view_copies().filter(team_id=team_id, id__in=list(copy_ids))- readers = _user_readers_of(copies)- if readers:- database = Database.create_for(- team_id=team_id,- bypass_warehouse_access_control=True,- allowed_system_tables=DATA_MODELING_ALLOWED_SYSTEM_TABLES,- )- for reader in readers:- _repoint_reader(team_id, reader, database)- for dag in DAG.objects.filter(team_id=team_id, id__in=copies.values("dag_id")).order_by("id"):- _delete_unread_copies(team_id, dag, copies)diff --git a/products/data_modeling/backend/management/commands/backfill_dag_schedule_type.py b/products/data_modeling/backend/management/commands/backfill_dag_schedule_type.pydeleted file mode 100644index e1c26c7d9872..000000000000--- a/products/data_modeling/backend/management/commands/backfill_dag_schedule_type.py+++ /dev/null@@ -1,79 +0,0 @@-from django.core.management.base import BaseCommand--import structlog-from asgiref.sync import async_to_sync-from temporalio.client import ScheduleListActionStartWorkflow, ScheduleUpdate, ScheduleUpdateInput-from temporalio.common import SearchAttributePair--from posthog.temporal.common.client import async_connect-from posthog.temporal.common.search_attributes import POSTHOG_SCHEDULE_TYPE_KEY--from products.data_modeling.backend.schedule import DATA_MODELING_EXECUTE_DAG_WORKFLOW--logger = structlog.get_logger(__name__)---class Command(BaseCommand):- help = "Stamp PostHogScheduleType on existing v2 execute-dag schedules so they can be found by a server-side filter"-- def add_arguments(self, parser):- parser.add_argument(- "--dry-run",- action="store_true",- default=False,- help="Only report how many schedules would be stamped, without updating any",- )-- def handle(self, *args, **options):- dry_run = options["dry_run"]- found, already_stamped, updated, failed = _backfill_schedule_type(dry_run)- verb = "would stamp" if dry_run else "stamped"- stamped = found - already_stamped if dry_run else updated- logger.info(- "Backfill complete",- execute_dag_schedules=found,- already_stamped=already_stamped,- updated=updated,- failed=failed,- dry_run=dry_run,- )- self.stdout.write(- f"execute-dag schedules found: {found}; already stamped: {already_stamped}; "- f"{verb}: {stamped}; failed: {failed}"- )---async def _updater(input: ScheduleUpdateInput) -> ScheduleUpdate:- merged = input.description.typed_search_attributes.updated(- SearchAttributePair(key=POSTHOG_SCHEDULE_TYPE_KEY, value=DATA_MODELING_EXECUTE_DAG_WORKFLOW)- )- return ScheduleUpdate(schedule=input.description.schedule, search_attributes=merged)---@async_to_sync-async def _backfill_schedule_type(dry_run: bool) -> tuple[int, int, int, int]:- temporal = await async_connect()- found = 0- already_stamped = 0- updated = 0- failed = 0- async for listing in await temporal.list_schedules():- action = listing.schedule.action if listing.schedule else None- if not (- isinstance(action, ScheduleListActionStartWorkflow)- and action.workflow == DATA_MODELING_EXECUTE_DAG_WORKFLOW- ):- continue- found += 1- if listing.typed_search_attributes.get(POSTHOG_SCHEDULE_TYPE_KEY) == DATA_MODELING_EXECUTE_DAG_WORKFLOW:- already_stamped += 1- continue- if dry_run:- continue- try:- await temporal.get_schedule_handle(listing.id).update(_updater)- updated += 1- except Exception:- failed += 1- logger.exception("Error stamping schedule", schedule_id=listing.id)- return found, already_stamped, updated, faileddiff --git a/products/data_modeling/backend/management/commands/backfill_missing_model_paths.py b/products/data_modeling/backend/management/commands/backfill_missing_model_paths.pydeleted file mode 100644index 1274b5684c7c..000000000000--- a/products/data_modeling/backend/management/commands/backfill_missing_model_paths.py+++ /dev/null@@ -1,137 +0,0 @@-"""This command will soon become obsolete as we migrate every materialization to v2 backend."""--import time-import logging--from django.conf import settings-from django.core.management.base import BaseCommand, CommandError--import structlog--from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery-from products.data_modeling.backend.models.modeling import DataWarehouseModelPath--logger = structlog.get_logger(__name__)---class Command(BaseCommand):- help = (- "Creates missing DataWarehouseModelPath records for materialized saved queries that have schedules but no paths"- )-- def add_arguments(self, parser):- parser.add_argument(- "--team-ids",- default=None,- type=str,- help="Comma separated list of team IDs to filter by",- )- parser.add_argument(- "--start-after-team-id",- default=None,- type=int,- help="Resume after this team ID (exclusive), following the team_id ordering used by this command",- )- parser.add_argument(- "--batch-size",- default=50,- type=int,- help="Number of saved queries to process per batch (default: 50)",- )- parser.add_argument(- "--batch-delay",- default=1.0,- type=float,- help="Seconds to wait between batches (default: 1.0)",- )- parser.add_argument(- "--dry-run",- action="store_true",- default=False,- help="Only show what would be updated without making changes",- )-- def handle(self, **options):- logger.setLevel(logging.INFO)-- queryset = DataWarehouseSavedQuery.objects.filter(- sync_frequency_interval__isnull=False,- ).exclude(deleted=True)-- if options.get("team_ids") is not None:- try:- team_ids = [int(tid) for tid in options["team_ids"].split(",")]- except ValueError:- raise CommandError("team-ids must be a comma separated list of team IDs")- queryset = queryset.filter(team_id__in=team_ids)-- if options.get("start_after_team_id") is not None:- queryset = queryset.filter(team_id__gt=options["start_after_team_id"])-- saved_queries = list(queryset.order_by("team_id", "id"))-- if not saved_queries:- logger.info("No materialized saved queries found matching filters")- return-- logger.info(f"Checking {len(saved_queries)} materialized saved queries for missing model paths")-- sq_ids_with_paths = set(- DataWarehouseModelPath.objects.filter(saved_query__in=saved_queries).values_list(- "saved_query_id", flat=True- )- )- missing = [sq for sq in saved_queries if sq.id not in sq_ids_with_paths]-- logger.info(f"Found {len(missing)} saved queries with no model paths (out of {len(saved_queries)} total)")-- if not missing:- logger.info("All saved queries have model paths, nothing to do")- return-- if options["dry_run"]:- for sq in missing:- logger.info("Would create model paths", saved_query_id=str(sq.id), team_id=sq.team_id, name=sq.name)- return-- if not settings.TEST:- confirm = input(f"\n\tWill create model paths for {len(missing)} saved queries. Proceed? (y/n) ")- if confirm.strip().lower() != "y":- logger.info("Aborting")- return-- batch_size = options["batch_size"]- batch_delay = options["batch_delay"]- created = 0- failed = 0-- for batch_start in range(0, len(missing), batch_size):- batch = missing[batch_start : batch_start + batch_size]- batch_num = batch_start // batch_size + 1- total_batches = (len(missing) + batch_size - 1) // batch_size-- logger.info(f"Processing batch {batch_num}/{total_batches} ({len(batch)} queries)")-- for sq in batch:- try:- sq.setup_model_paths()- created += 1- logger.info(- "Created model paths",- saved_query_id=str(sq.id),- team_id=sq.team_id,- name=sq.name,- )- except Exception:- failed += 1- logger.exception(- "Error creating model paths",- saved_query_id=str(sq.id),- team_id=sq.team_id,- )-- if batch_start + batch_size < len(missing):- logger.info(f"Sleeping {batch_delay}s between batches")- time.sleep(batch_delay)-- logger.info(f"Done! Created: {created}, Failed: {failed}, Total checked: {len(saved_queries)}")diff --git a/products/data_modeling/backend/management/commands/backfill_saved_query_nodes.py b/products/data_modeling/backend/management/commands/backfill_saved_query_nodes.pydeleted file mode 100644index 9ac53885312b..000000000000--- a/products/data_modeling/backend/management/commands/backfill_saved_query_nodes.py+++ /dev/null@@ -1,156 +0,0 @@-from collections.abc import Iterable-from datetime import timedelta-from itertools import groupby--from django.core.exceptions import ObjectDoesNotExist-from django.core.management.base import BaseCommand, CommandParser-from django.db import transaction-from django.db.models import Exists, OuterRef--from posthog.hogql.database.database import Database-from posthog.hogql.errors import BaseHogQLError--from posthog.dataclasses import frozen-from posthog.models import Team--from products.data_modeling.backend.logic.node_frequency import persist_seed_targets-from products.data_modeling.backend.logic.saved_query_dag_sync import (- adopt_saved_query_without_edges,- lineage_database,- sync_saved_query_to_dag,-)-from products.data_modeling.backend.logic.schedule_reconcile import reconcile_dag_schedules-from products.data_modeling.backend.logic.tier_run_report import MATERIALIZING_TYPES-from products.data_modeling.backend.models.dag import DAG-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery-from products.data_modeling.backend.models.modeling import UnknownParentError-from products.data_modeling.backend.models.node import Node--DEFAULT_MATERIALIZED_TARGET = timedelta(days=1)-ADOPTABLE_SYNC_ERRORS = (BaseHogQLError, UnknownParentError, ObjectDoesNotExist)---@frozen-class TeamBackfill:- synced: int- adopted: int- seeded: int- failed: dict[str, str]---class Command(BaseCommand):- help = (- "Give every live saved query that has no DAG node a node in its team's default DAG. "- "A query whose lineage does not resolve gets a node with no edges and a sync-failed marker. "- "A materialized query gets a freshness target, its own interval or else its DAG's or else 1 day, "- "so a tier schedule picks it up. Any other sync error is reported and left for a re-run. "- "Each team's DAG schedules are reconciled afterwards, and always for a team passed with --team-id. "- "Managed-viewset queries are skipped. Dry run unless --apply is passed."- )-- def add_arguments(self, parser: CommandParser) -> None:- parser.add_argument("--team-id", type=int, action="append", dest="team_ids", help="Limit to this team.")- parser.add_argument("--apply", action="store_true", help="Write the nodes. Without it, only list them.")-- def handle(self, *args: object, team_ids: list[int] | None, apply: bool, **options: object) -> None:- nodeless = nodeless_saved_queries(team_ids)- teams = sorted({saved_query.team_id for saved_query in nodeless})- self.stdout.write(f"{len(nodeless)} nodeless saved queries across {len(teams)} teams")- if not apply:- for saved_query in nodeless:- self.stdout.write(f" team {saved_query.team_id}: {saved_query.name} ({saved_query.id})")- return-- dags_to_reconcile = {- team.pk: DAG.get_or_create_default(team) for team in Team.objects.filter(id__in=team_ids or [])- }- for team_id, grouped in groupby(nodeless, key=lambda saved_query: saved_query.team_id):- team_queries = list(grouped)- dag = DAG.get_or_create_default(team_queries[0].team)- self._report(team_id, backfill_team(team_queries, dag))- dags_to_reconcile[team_id] = dag- for team_id, dag in dags_to_reconcile.items():- self._reconcile(team_id, dag)-- def _report(self, team_id: int, result: TeamBackfill) -> None:- for name, error in result.failed.items():- self.stderr.write(f" team {team_id}: FAILED {name}, re-run to retry: {error}")- self.stdout.write(- f" team {team_id}: synced {result.synced}, adopted without edges {result.adopted}, "- f"failed {len(result.failed)}, seeded targets {result.seeded}"- )-- def _reconcile(self, team_id: int, dag: DAG) -> None:- try:- applied = reconcile_dag_schedules(dag, require_tiered=True)- except Exception as error:- self.stderr.write(- f" team {team_id}: schedule reconcile FAILED, re-run with --team-id {team_id} --apply: {error!r}"- )- return- if not applied:- self.stderr.write(f" team {team_id}: schedules not reconciled, the DAG still runs on a pre-tier schedule")---def nodeless_saved_queries(team_ids: list[int] | None) -> list[DataWarehouseSavedQuery]:- queryset = (- DataWarehouseSavedQuery.objects.exclude(deleted=True)- .filter(managed_viewset__isnull=True)- .exclude(Exists(Node.objects.filter(saved_query_id=OuterRef("pk"))))- .select_related("team")- .order_by("team_id", "created_at")- )- if team_ids:- queryset = queryset.filter(team_id__in=team_ids)- return list(queryset)---def backfill_team(saved_queries: list[DataWarehouseSavedQuery], dag: DAG) -> TeamBackfill:- database = lineage_database(dag.team)- with transaction.atomic():- synced, unresolved = sync_until_stuck(saved_queries, dag, database)- adoptable = {- saved_query: error for saved_query, error in unresolved.items() if isinstance(error, ADOPTABLE_SYNC_ERRORS)- }- for saved_query, error in adoptable.items():- adopt_saved_query_without_edges(saved_query.team_id, saved_query.id, str(error), dag_id=dag.id)- # A child that only failed because its parent had no node can sync now that the parent has one.- resynced, _ = sync_until_stuck(list(adoptable), dag, database)- synced.extend(resynced)- adoptable = {saved_query: error for saved_query, error in adoptable.items() if saved_query not in resynced}- materializing_node_ids = Node.objects.filter(- dag=dag, saved_query__in=[*synced, *adoptable], type__in=MATERIALIZING_TYPES- ).values_list("id", flat=True)- seeded = persist_seed_targets(- dag,- default=DEFAULT_MATERIALIZED_TARGET,- only_node_ids={str(node_id) for node_id in materializing_node_ids},- )- return TeamBackfill(- synced=len(synced),- adopted=len(adoptable),- seeded=seeded,- failed={- saved_query.name: repr(error) for saved_query, error in unresolved.items() if saved_query not in adoptable- },- )---def sync_until_stuck(- saved_queries: Iterable[DataWarehouseSavedQuery], dag: DAG, database: Database-) -> tuple[list[DataWarehouseSavedQuery], dict[DataWarehouseSavedQuery, Exception]]:- pending = list(saved_queries)- synced: list[DataWarehouseSavedQuery] = []- while True:- failures: dict[DataWarehouseSavedQuery, Exception] = {}- for saved_query in pending:- try:- with transaction.atomic():- sync_saved_query_to_dag(saved_query, dag=dag, reconcile=False, database=database)- except Exception as error:- failures[saved_query] = error- continue- synced.append(saved_query)- if not failures or len(failures) == len(pending):- return synced, failures- pending = list(failures)diff --git a/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py b/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.pyindex 7ddef4da482e..28833a2d3a0c 100644--- a/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py+++ b/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py@@ -56,8 +56,7 @@ def has_live_dependents(saved_query: DataWarehouseSavedQuery) -> bool:`delete_node_from_dag` refuses the delete on the same condition. This is the cheap pre-filterfor it: a query with a live dependent is skipped before the delete takes an advisory lock on- every DAG the query has a node in. Nodes in several DAGs is the population `consolidate_dags`- exists for.+ every DAG the query has a node in."""nodes = Node.objects.filter(team_id=saved_query.team_id, saved_query=saved_query)return (diff --git a/products/data_modeling/backend/management/commands/consolidate_dags.py b/products/data_modeling/backend/management/commands/consolidate_dags.pydeleted file mode 100644index 113523172999..000000000000--- a/products/data_modeling/backend/management/commands/consolidate_dags.py+++ /dev/null@@ -1,849 +0,0 @@-import uuid-import dataclasses-from collections import defaultdict, deque-from datetime import timedelta-from typing import Any--from django.conf import settings-from django.core.exceptions import ObjectDoesNotExist-from django.core.management.base import BaseCommand, CommandError, CommandParser-from django.db import transaction-from django.db.models import QuerySet--import structlog-from temporalio.client import Client--from posthog.hogql.errors import BaseHogQLError--from posthog.models.team import Team-from posthog.temporal.common.client import sync_connect--from products.data_modeling.backend.logic.cohort_scheduling import is_tier_schedule_id-from products.data_modeling.backend.logic.freshness import (- STREAMING,- all_source_floors,- format_cadence,- normalize_seed_target,-)-from products.data_modeling.backend.logic.node_frequency import (- build_frequency_graph,- get_declared_target,- schedulable_nodes,- set_declared_target,-)-from products.data_modeling.backend.logic.saved_query_dag_sync import (- move_node_out_of_managed_dag,- node_type_for,- sync_saved_query_to_dag,-)-from products.data_modeling.backend.logic.saved_query_reads import PROXY_SAVED_QUERY_ID_PROPERTY-from products.data_modeling.backend.logic.schedule_reconcile import (- delete_dag_schedules,- delete_v1_saved_query_schedules,- list_existing_schedule_ids,- null_saved_query_intervals,- reconcile_dag_schedules,-)-from products.data_modeling.backend.models.dag import DAG, DEFAULT_DAG_NAME, REVENUE_ANALYTICS_DAG_NAME-from products.data_modeling.backend.models.data_modeling_job import DataModelingJob, DataModelingJobStatus-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery-from products.data_modeling.backend.models.edge import Edge-from products.data_modeling.backend.models.modeling import UnknownParentError-from products.data_modeling.backend.models.node import Node, NodeType--logger = structlog.get_logger(__name__)--# How the target DAG is currently scheduled — decides which Temporal work is safe.-MODE_TIERED = "tiered" # per-cadence tier schedules ({dag_id}:{seconds})-MODE_LEGACY_V2 = "legacy-v2" # single whole-DAG execute-dag schedule (id == dag_id)-MODE_V1 = "v1-only" # no execute-dag schedule; queries run on per-query v1 schedules--# The sync failures --adopt-unresolvable may adopt: the query's SQL failing to parse or resolve-# (BaseHogQLError covers parse errors and the BoundedResolver family), a parent that is not a-# known table or view, or a parent whose node lives in a DAG the resolver cannot see (surfaces-# as Node/SavedQuery DoesNotExist). Operational failures (DB errors, ManagedDAGError, a query-# with no SQL at all) stay failed moves — a retry could still move those intact.-ADOPTABLE_SYNC_ERRORS = (BaseHogQLError, UnknownParentError, ObjectDoesNotExist)---class MoveItem:- saved_query: DataWarehouseSavedQuery- declared_target: timedelta | None---class DropItem:- saved_query_id: str- node_id: str- name: str- declared_target: timedelta | None---class SourceDagPlan:- dag: DAG- moves: list[MoveItem] # dependency (topological) order- drops: list[DropItem]- moved_node_ids: set[str] # kept for the dry run's edge-impact count- dropped_node_ids: set[str]---class Command(BaseCommand):- help = (- "Plan and (with --apply) execute collapsing a team's overlapping DAGs into one. The dry run "- "is the planner: it partitions each source DAG's nodes into MOVE/DROP, counts the edges a "- "merge would re-point or dedup, flags saved queries whose declared freshness targets differ "- "between copies (a 'which cadence wins?' call for a human), and checks for anomalous "- "cross-DAG edges. For every non-managed source DAG, saved queries already in the "- "target are dropped (their duplicate node dies with the source DAG) and the rest are moved "- "by re-syncing them into the target in dependency order, then the source DAG is deleted "- "(cascading its nodes and edges). Tears down the source DAGs' execute-dag schedules, then "- "finalizes the target from its persistent state: seed declared targets from leftover v1 "- "intervals, sweep v1 per-query schedules, null the intervals, reconcile tiers once. "- "Because the finalize derives from the target rather than this run's moves, re-running "- "with --apply completes a partially-applied consolidation. A query whose SQL fails "- "dependency resolution normally keeps its source DAG and fails the run; "- "--adopt-unresolvable instead lands it in the target with no edges and a "- "properties.system.degraded_sync marker. A system-managed DAG (Revenue Analytics) is "- "folded in as a source too: its schedules are torn down before its nodes move, and the "- "proxy table nodes that stood in for its views are replaced by the real nodes. "- "--teams-with-managed-dag runs the fold for every team that still has a managed DAG and "- "halts at the first team that fails. Dry run by default; pass --apply to execute."- )-- def add_arguments(self, parser: CommandParser) -> None:- teams = parser.add_mutually_exclusive_group(required=True)- teams.add_argument("--team-id", type=int)- teams.add_argument(- "--teams-with-managed-dag",- action="store_true",- help="Run for every team that still has a system-managed DAG, halting at the first failure",- )- parser.add_argument(- "--target-dag-id",- type=str,- default=None,- help="Force the merge target DAG (default: the canonical 'Default' DAG, else the largest)",- )- parser.add_argument(- "--apply",- action="store_true",- default=False,- help="Execute the consolidation. Without this flag the command is a dry run and changes nothing.",- )- parser.add_argument("--yes", action="store_true", default=False, help="Skip the confirmation prompt")- parser.add_argument(- "--verbose",- action="store_true",- help="List per-query names under each source DAG's move/drop counts (default: counts only)",- )- parser.add_argument(- "--adopt-unresolvable",- action="store_true",- default=False,- help=(- "When a query's SQL fails dependency resolution during a move, create its node in "- "the target with no incoming edges (marked properties.system.degraded_sync) instead "- "of keeping the source DAG and failing the run. The query keeps failing at "- "materialization time as it already does; edges rebuild on its next successful "- "sync. Caveat: until then the node runs at level 0, before any real parents."- ),- )-- def handle(self, *args: Any, **options: Any) -> None:- if options["teams_with_managed_dag"]:- self._fold_managed_dags_fleet(options)- return- self._consolidate_team(options["team_id"], options)-- def _fold_managed_dags_fleet(self, options: dict[str, Any]) -> None:- team_ids = list(- DAG.objects.filter(name=REVENUE_ANALYTICS_DAG_NAME)- .order_by("team_id")- .values_list("team_id", flat=True)- .distinct()- )- self.stdout.write(f"{len(team_ids)} team(s) have a system-managed DAG")- for processed, team_id in enumerate(team_ids):- try:- self._consolidate_team(team_id, options)- except Exception as error:- logger.exception("Failed to fold the managed DAG of a team", team_id=team_id)- raise CommandError(- f"halted at team {team_id} after {processed} team(s): {error}; "- "fix it and re-run, finished teams no longer have a managed DAG"- )- self.stdout.write(f"\nProcessed {len(team_ids)} team(s)")-- def _consolidate_team(self, team_id: int, options: dict[str, Any]) -> None:- apply_changes: bool = options["apply"]- all_dags = list(DAG.objects.filter(team_id=team_id))- if not all_dags:- raise CommandError(f"Team {team_id} has no DAGs")-- managed_dags = [dag for dag in all_dags if dag.is_managed]- dags = [dag for dag in all_dags if not dag.is_managed]-- forced_id: str | None = options["target_dag_id"]- if forced_id:- try:- forced_uuid = uuid.UUID(forced_id)- except ValueError:- raise CommandError(f"--target-dag-id must be a UUID, got {forced_id!r}")- if any(dag.id == forced_uuid for dag in managed_dags):- raise CommandError(f"--target-dag-id {forced_id} is a system-managed DAG and cannot be a merge target")-- self.stdout.write(f"\nTeam {team_id} — {len(all_dags)} DAG(s)")- if managed_dags:- folded = ", ".join(f"{dag.name} ({dag.id})" for dag in managed_dags)- self.stdout.write(f"Folding {len(managed_dags)} system-managed DAG(s) into the target: {folded}")-- if not dags and not forced_id:- if not apply_changes:- self.stdout.write(- "No Default DAG yet — --apply creates it as the merge target (dry run: nothing changed)"- )- return- target = DAG.get_or_create_default(Team.objects.get(id=team_id))- else:- target = self._pick_target(dags, forced_id)- source_dags = sorted(- (dag for dag in all_dags if dag.id != target.id), key=lambda dag: (not dag.is_managed, dag.name)- )- self.stdout.write(f"Merge target: {target.name} ({target.id}) — {self._target_reason(target, options)}")-- plans = self._build_plan(target, source_dags)- self._print_plan(target, plans, team_id=team_id, dags=all_dags, verbose=options["verbose"])-- if not apply_changes:- self.stdout.write(- "\n(dry run — nothing was changed; target schedule mode is detected at apply time; "- "pass --apply to execute)"- )- return-- if not settings.TEST and not options["yes"]:- confirm = input(- f"\n\tWill consolidate {len(source_dags)} source DAG(s) into {target.name} ({target.id}) "- f"for team {team_id}, deleting the source DAGs. Proceed? (y/n) "- )- if confirm.strip().lower() != "y":- self.stdout.write("Aborting")- return-- # Connect before mutating anything: schedule teardown is mandatory once DAGs start moving,- # so a Temporal outage must abort the run while it is still a pure no-op.- try:- temporal = sync_connect()- except Exception as error:- raise CommandError(f"Cannot connect to Temporal, aborting before any change: {error}")- mode = self._detect_target_mode(self._list_dag_schedules_or_abort(str(target.id)))- self.stdout.write(f"\nTarget schedule mode: {mode}")- if mode == MODE_V1:- # A v2-scheduled source would strand its queries: its execute-dag schedules die with- # the source DAG, but a v1-only target has no schedule to hand coverage to.- v2_sources = [- f"{plan.dag.name} ({plan.dag.id})"- for plan in plans- if not plan.dag.is_managed and self._list_dag_schedules_or_abort(str(plan.dag.id))- ]- if v2_sources:- raise CommandError(- f"target {target.name} ({target.id}) is v1-only but source(s) "- f"{', '.join(v2_sources)} are v2-scheduled — migrate the target to tiers first "- "via reconcile_freshness_schedules, or choose a tiered target"- )- self.stdout.write(- " target has no execute-dag schedule and every source is v1-only — graph-only "- "consolidation: v1 per-query schedules and sync intervals are kept so the moved "- "queries stay scheduled"- )-- consolidated_sq_ids: set[str] = set()- kept_dags: list[str] = []- adopted_names: list[str] = []- for plan in plans:- kept_reason = self._apply_source_dag(- plan,- target,- temporal,- consolidated_sq_ids,- adopted_names,- adopt_unresolvable=options["adopt_unresolvable"],- )- if kept_reason:- kept_dags.append(f"{plan.dag.name} ({plan.dag.id}): {kept_reason}")-- if not kept_dags and target.name != DEFAULT_DAG_NAME:- self._rename_to_default(target)- proxies_left = self._replace_proxy_nodes(target)- try:- v1_failed_sq_ids = self._finalize_target(target, temporal, bootstrap_tiers=bool(managed_dags))- except Exception as error:- if not managed_dags or kept_dags:- raise- # The managed DAG is what makes a run create the first tier schedules, and it is gone now.- raise CommandError(- f"{error}; re-running will not create the target's first tier schedules because the "- f"managed DAG is gone, so run reconcile_freshness_schedules --team-id {team_id} to converge them"- ) from error-- self.stdout.write(f"\nDone: consolidated {len(consolidated_sq_ids)} saved query(ies) into {target.name}")- if adopted_names:- self.stdout.write(- self.style.WARNING(- f" adopted {len(adopted_names)} query(ies) without edges (SQL failed to resolve): "- f"{', '.join(sorted(adopted_names))} — they run at level 0 until their next "- "successful sync rebuilds edges"- )- )- if v1_failed_sq_ids:- self.stdout.write(- self.style.WARNING(- f" v1 schedule delete failed for {len(v1_failed_sq_ids)} saved query(ies) "- f"(intervals kept for retry): {', '.join(sorted(v1_failed_sq_ids))}"- )- )- if kept_dags:- for line in kept_dags:- self.stdout.write(self.style.ERROR(f" kept source DAG {line}"))- raise CommandError(- f"consolidation incomplete: {len(kept_dags)} source DAG(s) kept; "- "resolve the failures above and re-run (the command converges on re-run)"- )- if proxies_left:- raise CommandError(- f"consolidation incomplete: proxy node(s) {', '.join(sorted(proxies_left))} still stand in for "- "views whose node never reached the target; re-run once those views have moved"- )-- def _pick_target(self, dags: list[DAG], target_dag_id: str | None) -> DAG:- if target_dag_id:- try:- wanted = uuid.UUID(target_dag_id)- except ValueError:- raise CommandError(f"--target-dag-id must be a UUID, got {target_dag_id!r}")- for dag in dags:- if dag.id == wanted:- return dag- raise CommandError(f"--target-dag-id {target_dag_id} is not a DAG of this team")-- for dag in dags:- if dag.name == DEFAULT_DAG_NAME:- return dag-- counts: dict[Any, int] = defaultdict(int)- for dag_id in Node.objects.filter(dag__in=dags).values_list("dag_id", flat=True):- counts[dag_id] += 1- return sorted(dags, key=lambda d: (-counts.get(d.id, 0), d.name))[0]-- def _target_reason(self, target: DAG, options: dict[str, Any]) -> str:- if options["target_dag_id"]:- return "chosen via --target-dag-id"- if target.name == DEFAULT_DAG_NAME:- return "canonical 'Default' DAG"- return "no Default DAG — largest DAG by node count"-- def _build_plan(self, target: DAG, source_dags: list[DAG]) -> list[SourceDagPlan]:- """Partition every source DAG's schedulable nodes into MOVE/DROP and- topologically order the moves, all read-only so any cycle aborts before a single mutation.- """- target_sq_ids = {str(sq_id) for sq_id in schedulable_nodes(target).values_list("saved_query_id", flat=True)}- claimed: set[str] = set() # saved queries a preceding source DAG already moves- placed_in_managed_dag = {- str(sq_id)- for sq_id in Node.objects.filter(- team=target.team, dag__name=REVENUE_ANALYTICS_DAG_NAME, saved_query__isnull=False- ).values_list("saved_query_id", flat=True)- }- plans: list[SourceDagPlan] = []- for dag in source_dags:- moves_by_node: dict[str, MoveItem] = {}- drops: list[DropItem] = []- floors = self._source_floors(dag) if dag.sync_frequency_interval is not None else {}- for node in schedulable_nodes(dag).select_related("saved_query"):- saved_query = node.saved_query- if saved_query is None or (not dag.is_managed and str(saved_query.id) in placed_in_managed_dag):- continue- sq_id = str(saved_query.id)- # Query-interval intent survives the move on the saved query itself, and a moved- # no-interval query adopts the target DAG's default cadence — both seeded in the- # finalize. What has no carrier is the case where the target has no default either:- # cadence then lives only on the *source* DAG's interval, which the finalize can't- # see, so capture it as a declared target before the source DAG — the only place it- # exists — is deleted; without it the moved node has no effective cadence and- # reconcile drops it from every tier.- declared = get_declared_target(node)- if (- declared is None- and saved_query.sync_frequency_interval is None- and target.sync_frequency_interval is None- and dag.sync_frequency_interval is not None- ):- declared = normalize_seed_target(dag.sync_frequency_interval, floors.get(str(node.id), STREAMING))- if sq_id in target_sq_ids or sq_id in claimed:- drops.append(- DropItem(saved_query_id=sq_id, node_id=str(node.id), name=node.name, declared_target=declared)- )- else:- moves_by_node[str(node.id)] = MoveItem(saved_query=saved_query, declared_target=declared)- claimed.add(sq_id)- order = self._dependency_order(dag, set(moves_by_node))- plans.append(- SourceDagPlan(- dag=dag,- moves=[moves_by_node[node_id] for node_id in order],- drops=drops,- moved_node_ids=set(moves_by_node),- dropped_node_ids={drop.node_id for drop in drops},- )- )- return plans-- def _source_floors(self, dag: DAG) -> dict[str, timedelta]:- graph = build_frequency_graph(dag)- return all_source_floors(graph.edges, graph.source_intervals)-- def _cadence_intent(self, node: Node, dag: DAG, floors: dict[str, timedelta]) -> timedelta | None:- """A node's cadence intent, resolved in the same order as `seed_targets`.-- The DAG-interval fallback matters because the finalize nulls every query interval before- reconciling: a query that was only ever scheduled by its DAG's default cadence would then- have no declared target and no interval, and `reconcile_dag_schedules` drops a node with no- effective cadence from every tier — it would stop materializing with nothing left to show why.- """- interval = None- if node.saved_query is not None and node.saved_query.sync_frequency_interval is not None:- interval = node.saved_query.sync_frequency_interval- elif dag.sync_frequency_interval is not None:- interval = dag.sync_frequency_interval- if interval is None:- return None- return normalize_seed_target(interval, floors.get(str(node.id), STREAMING))-- def _dependency_order(self, dag: DAG, node_ids: set[str]) -> list[str]:- """Kahn's algorithm over the source DAG's edges restricted to the moved nodes: a moved- query's node must exist in the target before a moved dependent's sync resolves it. The- model forbids cycles, but a corrupt graph must abort here, before anything is mutated.- """- children: dict[str, list[str]] = defaultdict(list)- in_degree = dict.fromkeys(node_ids, 0)- for source_id, target_id in Edge.objects.filter(dag=dag).values_list("source_id", "target_id"):- src, tgt = str(source_id), str(target_id)- if src in node_ids and tgt in node_ids:- children[src].append(tgt)- in_degree[tgt] += 1- queue = deque(sorted(node_id for node_id, degree in in_degree.items() if degree == 0))- order: list[str] = []- while queue:- node_id = queue.popleft()- order.append(node_id)- for child in children[node_id]:- in_degree[child] -= 1- if in_degree[child] == 0:- queue.append(child)- if len(order) != len(node_ids):- raise CommandError(f"cycle detected among nodes to move in DAG {dag.id}; aborting before any change")- return order-- def _print_plan(- self, target: DAG, plans: list[SourceDagPlan], *, team_id: int, dags: list[DAG], verbose: bool- ) -> None:- if not plans:- self.stdout.write(" Only one DAG — nothing to consolidate.")- return- total_moved = 0- total_dropped = 0- total_repoint_edges = 0- total_dedup_edges = 0- for plan in plans:- total_moved += len(plan.moves)- total_dropped += len(plan.drops)- repoint, dedup = self._count_edge_impact(plan.dag, plan.moved_node_ids, plan.dropped_node_ids)- total_repoint_edges += repoint- total_dedup_edges += dedup- self.stdout.write(f"\n Source DAG {plan.dag.name} ({plan.dag.id}):")- self.stdout.write(f" move {len(plan.moves)}, drop {len(plan.drops)}")- self.stdout.write(f" edges to re-point (touch a moved node): {repoint}")- self.stdout.write(f" edges to drop as redundant (only dropped nodes): {dedup}")- self.stdout.write(- f" v1 schedules swept in the target-wide finalize (if target is v2-scheduled): "- f"{len(plan.moves) + len(plan.drops)}"- )- self.stdout.write(f" execute-dag schedules to tear down: all listed for DAG {plan.dag.id}")- if verbose:- if plan.moves:- move_names = ", ".join(item.saved_query.name for item in plan.moves)- self.stdout.write(f" move (in dependency order): {move_names}")- if plan.drops:- self.stdout.write(f" drop: {', '.join(sorted(drop.name for drop in plan.drops))}")-- conflicts = self._find_target_conflicts(dags)- self._print_conflicts(conflicts)- cross_dag_edges = self._count_cross_dag_edges(team_id)-- target_after = schedulable_nodes(target).count() + total_moved- self.stdout.write(f"\nSummary for target {target.name} ({target.id}):")- self.stdout.write(f" nodes in target after merge: {target_after}")- self.stdout.write(f" moved: {total_moved}, dropped: {total_dropped}")- self.stdout.write(f" edges to re-point: {total_repoint_edges}, edges to drop: {total_dedup_edges}")- self.stdout.write(f" source DAGs to delete: {len(plans)}")- self.stdout.write(f" proxy nodes to replace: {self._proxy_nodes(target).count()}")- self.stdout.write(f" freshness-target conflicts: {len(conflicts)}")- if cross_dag_edges:- self.stdout.write(self.style.WARNING(f" ⚠ anomalous cross-DAG edges found: {cross_dag_edges}"))- else:- self.stdout.write(" cross-DAG edges: 0 (edges are within-DAG by construction)")- self.stdout.write(" target reconciled once at the end (if on cadence tiers)")-- def _count_edge_impact(self, dag: DAG, moved_ids: set[str], dropped_ids: set[str]) -> tuple[int, int]:- """Count edges a real merge would touch. Edges are within-DAG (source/target/edge all share- the DAG), so an edge touching a moved node must be re-pointed to the target DAG; an edge- whose only involved nodes are dropped is redundant against the target's existing edge.- """- repoint = 0- dedup = 0- for source_id, target_id in Edge.objects.filter(dag=dag).values_list("source_id", "target_id"):- src, tgt = str(source_id), str(target_id)- if src in moved_ids or tgt in moved_ids:- repoint += 1- elif src in dropped_ids or tgt in dropped_ids:- dedup += 1- return repoint, dedup-- def _find_target_conflicts(self, dags: list[DAG]) -> list[tuple[str, list[tuple[str, timedelta | None]]]]:- """Saved queries with schedulable nodes in 2+ DAGs whose declared freshness targets differ.- A conflict is 2+ distinct non-None declared targets across the copies — after a merge only- one node survives, so a human must pick the winning cadence. All-unset or agreeing targets- are not conflicts (unset means 'no opinion', so a lone declared target wins cleanly).- """- by_saved_query: dict[object, list[tuple[str, str, timedelta | None]]] = defaultdict(list)- dag_names = {dag.id: dag.name for dag in dags}- for node in Node.objects.filter(dag__in=dags).exclude(type=NodeType.TABLE).exclude(saved_query__deleted=True):- if node.saved_query_id is None:- continue- by_saved_query[node.saved_query_id].append((dag_names[node.dag_id], node.name, get_declared_target(node)))-- conflicts: list[tuple[str, list[tuple[str, timedelta | None]]]] = []- for copies in by_saved_query.values():- if len(copies) < 2:- continue- declared = {target for _dag_name, _query_name, target in copies if target is not None}- if len(declared) < 2:- continue- query_name = copies[0][1]- copies_report = sorted((dag_name, target) for dag_name, _query_name, target in copies)- conflicts.append((query_name, copies_report))- return sorted(conflicts, key=lambda c: c[0])-- def _print_conflicts(self, conflicts: list[tuple[str, list[tuple[str, timedelta | None]]]]) -> None:- self.stdout.write("\nFreshness-target conflicts (which cadence wins after merge?):")- if not conflicts:- self.stdout.write(" none")- return- for query_name, copies in conflicts:- self.stdout.write(self.style.WARNING(f" ⚠ {query_name}:"))- for dag_name, target in copies:- label = format_cadence(target) if target is not None else "unset"- self.stdout.write(f" {dag_name}: {label}")-- def _count_cross_dag_edges(self, team_id: int) -> int:- """Defensive check: edges whose endpoints' DAGs disagree with the edge's own DAG. The model- forbids these, so a non-zero count means data corruption worth surfacing before any merge.- """- node_dags = dict(Node.objects.filter(team_id=team_id).values_list("id", "dag_id"))- anomalies = 0- for source_id, target_id, dag_id in Edge.objects.filter(team_id=team_id).values_list(- "source_id", "target_id", "dag_id"- ):- if node_dags.get(source_id) != dag_id or node_dags.get(target_id) != dag_id:- anomalies += 1- return anomalies-- def _list_dag_schedules_or_abort(self, dag_id: str) -> set[str]:- """Authoritative listing of a DAG's execute-dag schedule ids via the PostHogDagId search- attribute — never guessed from an id formula, so off-scheme schedules can't escape. Only- used before the first mutation, where aborting is still a pure no-op.- """- try:- return list_existing_schedule_ids(dag_id)- except Exception as error:- raise CommandError(f"Cannot list Temporal schedules for DAG {dag_id}, aborting before any change: {error}")-- def _detect_target_mode(self, target_schedule_ids: set[str]) -> str:- if any(is_tier_schedule_id(schedule_id) for schedule_id in target_schedule_ids):- return MODE_TIERED- if target_schedule_ids:- return MODE_LEGACY_V2- return MODE_V1-- def _apply_source_dag(- self,- plan: SourceDagPlan,- target: DAG,- temporal: Client,- consolidated_sq_ids: set[str],- adopted_names: list[str],- *,- adopt_unresolvable: bool = False,- ) -> str | None:- """Consolidate one source DAG. Returns None on success, or the reason the DAG was kept.-- Ordering, chosen so a crash at any point leaves a state a re-run converges from:- 1. (DB) sync each MOVE query into the target in dependency order — a crash here leaves- duplicates the re-run classifies as DROP.- 2. (Temporal) delete the source DAG's execute-dag schedules, from an authoritative- listing — idempotent, so a crash after this is retried harmlessly.- 3. (DB) delete the source DAG row, cascading its nodes and edges. This must come LAST:- the row is the only durable pointer to the DAG's Temporal schedules, so deleting it- before the teardown succeeded would orphan schedules forever.- The consolidated queries' v1 schedules are swept afterwards by the target-wide finalize,- which re-runs derive from the target's persistent state.- """- self.stdout.write(f"\n Source DAG {plan.dag.name} ({plan.dag.id}):")- teardown_before_moves = plan.dag.is_managed- if teardown_before_moves and self._has_running_materialization(plan):- return "a materialization of one of its views is running; re-run once it finishes"- if teardown_before_moves and not self._teardown_execute_dag_schedules(str(plan.dag.id)):- return "execute-dag schedule teardown failed before any node moved; re-run to retry it"- failed_moves: list[str] = []- moved_sq_ids: list[str] = []- for item in plan.moves:- try:- # Atomic, so a query whose dependencies fail to resolve leaves its node, edges and- # marker in the managed DAG and the re-run retries it instead of finding it moved.- with transaction.atomic():- if plan.dag.is_managed:- move_node_out_of_managed_dag(item.saved_query, target)- node = sync_saved_query_to_dag(item.saved_query, dag=target, reconcile=False)- except Exception as error:- adoptable = adopt_unresolvable and isinstance(error, ADOPTABLE_SYNC_ERRORS)- node = (- self._adopt_without_edges(item.saved_query, target, error, from_managed_dag=plan.dag.is_managed)- if adoptable- else None- )- if node is None:- failed_moves.append(item.saved_query.name)- logger.exception(- "Failed to move saved query into target DAG",- saved_query_id=str(item.saved_query.id),- source_dag_id=str(plan.dag.id),- target_dag_id=str(target.id),- )- self.stderr.write(f" FAILED to move {item.saved_query.name}: {error}")- continue- self._carry_declared_target(node, item.declared_target, item.saved_query.name)- moved_sq_ids.append(str(item.saved_query.id))- adopted_names.append(item.saved_query.name)- self.stdout.write(self.style.WARNING(f" adopted {item.saved_query.name} without edges: {error}"))- continue- if node is not None:- self._carry_declared_target(node, item.declared_target, item.saved_query.name)- moved_sq_ids.append(str(item.saved_query.id))- self.stdout.write(f" moved {item.saved_query.name}")- orphaned_drops: list[str] = []- for drop in plan.drops:- node = Node.objects.filter(dag=target, saved_query_id=drop.saved_query_id).first()- if node is None:- # The plan expected an earlier source DAG to have moved this query in. It did not,- # so the copy about to be cascaded away is the only place its declared target- # exists. Keep this DAG instead of destroying intent a re-run could still carry.- orphaned_drops.append(drop.name)- self.stderr.write(f" NOT dropping {drop.name}: its query never reached the target")- continue- if plan.dag.is_managed and drop.declared_target is not None:- set_declared_target(node, drop.declared_target)- else:- self._carry_declared_target(node, drop.declared_target, drop.name)- self.stdout.write(f" dropped duplicate {drop.name}")-- if failed_moves:- # The successful moves are now duplicates a re-run classifies as DROP; deleting the DAG- # here would destroy the only node of every query that failed to move.- return f"{len(failed_moves)} move(s) failed ({', '.join(failed_moves)})"- if orphaned_drops:- return f"{len(orphaned_drops)} duplicate(s) never reached the target ({', '.join(orphaned_drops)})"-- if not teardown_before_moves and not self._teardown_execute_dag_schedules(str(plan.dag.id)):- return "execute-dag schedule teardown failed; DAG kept so a re-run can retry it"-- plan.dag.delete()- self.stdout.write(f" deleted source DAG {plan.dag.id}")- consolidated_sq_ids.update(moved_sq_ids)- consolidated_sq_ids.update(drop.saved_query_id for drop in plan.drops)- return None-- def _adopt_without_edges(- self, saved_query: DataWarehouseSavedQuery, target: DAG, error: Exception, *, from_managed_dag: bool = False- ) -> Node | None:- """Last-resort landing for a query whose SQL will not resolve: create its node in the- target with no incoming edges so the consolidation can proceed and the source DAG can be- deleted. The query keeps failing at materialization time exactly as it did in the source- DAG (feeding the suspension circuit breaker), the marker makes these nodes findable- fleet-wide, and the next successful sync clears the marker and rebuilds real edges.- Returns None if even the bare node cannot be created, falling back to keeping the DAG.- """- try:- # Atomic so a failed marker save rolls the creation back too: a committed but- # unmarked node would make the re-run classify this query as DROP instead of- # retrying the move, stranding it edge-less and invisible to the marker sweep.- with transaction.atomic():- if from_managed_dag:- # The failed move rolled back, so bring the managed node over to keep its state.- move_node_out_of_managed_dag(saved_query, target)- node, _ = Node.objects.get_or_create(- team_id=saved_query.team_id,- saved_query=saved_query,- dag=target,- defaults={"name": saved_query.name, "type": node_type_for(saved_query)},- )- node.properties = node.properties or {}- node.mark_lineage_sync_failed(str(error))- node.save(update_fields=["properties"])- return node- except Exception:- logger.exception(- "Failed to adopt unresolvable saved query into target DAG",- saved_query_id=str(saved_query.id),- target_dag_id=str(target.id),- )- return None-- def _carry_declared_target(self, node: Node, declared: timedelta | None, name: str) -> None:- """Carry a source node's declared freshness target onto its target-DAG node. The target- DAG's copy wins a disagreement (the dry run surfaces these conflicts for a human first).- """- if declared is None:- return- existing = get_declared_target(node)- if existing is None:- set_declared_target(node, declared)- elif existing != declared:- self.stdout.write(- f" cadence conflict on {name}: keeping target's {format_cadence(existing)} "- f"(source declared {format_cadence(declared)})"- )-- def _teardown_execute_dag_schedules(self, dag_id: str) -> bool:- teardown = delete_dag_schedules(dag_id)- for schedule_id in teardown.deleted:- self.stdout.write(f" deleted execute-dag schedule {schedule_id}")- if not teardown.ok:- self.stderr.write(f" FAILED to tear down execute-dag schedules for DAG {dag_id}")- return teardown.ok-- def _finalize_target(self, target: DAG, temporal: Client, *, bootstrap_tiers: bool = False) -> set[str]:- """Converge the target after the moves: seed declared targets from leftover v1 intervals,- sweep the v1 per-query schedules, null the consumed intervals, and reconcile the tier- schedules once. Everything here derives from the target's persistent state — never this- run's move list — so re-running the command completes a consolidation a crash left- half-finished; that re-run IS the recovery mechanism, at the cost of a no-op run- re-sweeping and re-reconciling the target.-- The schedule mode is re-detected here rather than taken from the pre-flight check: a- concurrent reconcile can convert the target to tiers while the moves run, and acting on- the stale v1-only snapshot would skip the sweep and leave overlapping v1 + tier schedules.-- On a legacy-v2 target the whole-DAG schedule (node_ids=None) picks the moved nodes up- automatically, so only tiered targets are reconciled; the seeded declared targets preserve- cadence intent for the eventual tier conversion either way. On a v1-only target nothing- runs — every source was verified v1-only before any mutation, so the kept v1 schedules and- intervals keep the moved queries scheduled. Returns the saved-query ids whose v1 delete- failed (their intervals are kept as the retry signal).- """- try:- mode = self._detect_target_mode(list_existing_schedule_ids(str(target.id)))- except Exception as error:- raise CommandError(- f"consolidation applied but re-detecting target {target.id} schedule mode failed ({error}); "- "re-run with --apply to finish the finalize"- )- self.stdout.write(f"\n finalize: target schedule mode {mode}")- creates_first_tiers = mode == MODE_V1 and bootstrap_tiers- if mode not in (MODE_TIERED, MODE_LEGACY_V2) and not creates_first_tiers:- return set()- self._seed_targets_from_intervals(target)- nodes = list(schedulable_nodes(target).select_related("saved_query"))- v1_failed = delete_v1_saved_query_schedules(- nodes, team_id=target.team_id, dag_id=str(target.id), temporal=temporal- )- swept_ids = {str(node.saved_query_id) for node in nodes if node.saved_query_id is not None} - v1_failed- cleared = null_saved_query_intervals(target, only_saved_query_ids=swept_ids)- self.stdout.write(- f"\n swept v1 schedules for {len(nodes)} saved query(ies) ({len(v1_failed)} failed), "- f"cleared sync_frequency_interval on {cleared}"- )- if mode == MODE_TIERED or creates_first_tiers:- self._reconcile_target(target, require_tiered=not creates_first_tiers)- return v1_failed-- def _has_running_materialization(self, plan: SourceDagPlan) -> bool:- return DataModelingJob.objects.filter(- team_id=plan.dag.team_id,- status=DataModelingJobStatus.RUNNING,- saved_query_id__in=[item.saved_query.id for item in plan.moves]- + [drop.saved_query_id for drop in plan.drops],- ).exists()-- def _reconcile_target(self, target: DAG, *, require_tiered: bool) -> None:- try:- reconcile_dag_schedules(target, require_tiered=require_tiered)- self.stdout.write(f" reconciled target DAG {target.id} tier schedules")- except Exception as error:- logger.exception("Failed to reconcile target DAG after consolidation", dag_id=str(target.id))- raise CommandError(- f"consolidation applied but the final reconcile of target {target.id} failed ({error}); "- "run reconcile_freshness_schedules for this team to converge the tier schedules"- )-- def _rename_to_default(self, target: DAG) -> None:- """`DAG.get_or_create_default` looks the DAG up by name, so a team's only DAG must carry it."""- self.stdout.write(f"\n renamed target {target.name} ({target.id}) to {DEFAULT_DAG_NAME}")- target.name = DEFAULT_DAG_NAME- target.save(update_fields=["name"])-- def _proxy_nodes(self, dag: DAG) -> QuerySet[Node]:- return Node.objects.filter(dag=dag, type=NodeType.TABLE, properties__has_key=PROXY_SAVED_QUERY_ID_PROPERTY)-- def _replace_proxy_nodes(self, target: DAG) -> list[str]:- """Point each proxy's readers at the real node now in the target, then delete the proxy.-- Returns the names of the proxies kept because their view has no node in the target.- """- kept: list[str] = []- for proxy in self._proxy_nodes(target):- real = Node.objects.filter(- dag=target, saved_query_id=proxy.properties[PROXY_SAVED_QUERY_ID_PROPERTY]- ).first()- if real is None:- kept.append(proxy.name)- continue- with transaction.atomic():- for edge in Edge.objects.filter(dag=target, source=proxy):- Edge.objects.get_or_create(- team_id=target.team_id,- dag=target,- source=real,- target_id=edge.target_id,- defaults={"properties": edge.properties},- )- proxy.delete()- self.stdout.write(f" replaced proxy {proxy.name} with its view's node")- return kept-- def _seed_targets_from_intervals(self, target: DAG) -> None:- """A query about to lose its v1 sync_frequency_interval must not lose its cadence intent:- seed each target-DAG node's declared target from its query's interval (normalized, never- overwriting) before the interval is nulled — persist_seed_targets/convert_dag_to_tiers in- spirit, scoped to the target DAG and idempotent so crash-recovery re-runs converge.- """- floors = self._source_floors(target)- for node in schedulable_nodes(target).select_related("saved_query"):- if get_declared_target(node) is not None:- continue- seed = self._cadence_intent(node, target, floors)- if seed is None:- continue- set_declared_target(node, seed)diff --git a/products/data_modeling/backend/management/commands/preview_freshness_schedules.py b/products/data_modeling/backend/management/commands/preview_freshness_schedules.pyindex bd6e409f312f..7fbb1b80bd9c 100644--- a/products/data_modeling/backend/management/commands/preview_freshness_schedules.py+++ b/products/data_modeling/backend/management/commands/preview_freshness_schedules.py@@ -1,12 +1,11 @@import uuid-from collections import defaultdictfrom django.core.management.base import BaseCommand, CommandErrorfrom products.data_modeling.backend.logic.cohort_scheduling import format_tier, tier_sort_keyfrom products.data_modeling.backend.logic.schedule_reconcile import DagSchedulePreview, preview_dag_schedulesfrom products.data_modeling.backend.models.dag import DAG-from products.data_modeling.backend.models.node import Node, NodeType+from products.data_modeling.backend.models.node import Nodeclass Command(BaseCommand):@@ -14,8 +13,7 @@ class Command(BaseCommand):"Dry-run: show the per-cadence-tier schedules a DAG would get from its nodes' freshness ""targets, and the create/update/delete plan against its current schedules. Reads only — ""never creates, updates, or deletes a schedule, and touches no live read/write path. "- "Prints a compact summary by default (--verbose for per-node cadences), plus a team-level "- "cross-DAG duplication report to judge whether a redundant DAG is safe to drop."+ "Prints a compact summary by default (--verbose for per-node cadences).")def add_arguments(self, parser):@@ -44,7 +42,6 @@ class Command(BaseCommand):raise CommandError("No matching DAGs")for dag in dags:self._preview(dag, seed=options["seed"], verbose=options["verbose"])- self._print_cross_dag_duplication(options["team_id"])self.stdout.write("\n(dry run — no schedules were created, updated, or deleted)")def _preview(self, dag: DAG, *, seed: bool, verbose: bool) -> None:@@ -121,38 +118,3 @@ class Command(BaseCommand):self.stdout.write(f" UPDATE {schedule_id} ({format_tier(tier)}, {len(node_ids)} nodes)")for schedule_id in sorted(plan.to_delete):self.stdout.write(f" DELETE {schedule_id}")-- def _print_cross_dag_duplication(self, team_id: int) -> None:- """Which saved queries a team materializes in more than one DAG, and which are unique to- each DAG. A query in >1 DAG is materialized redundantly; a DAG whose queries all also live- elsewhere is safe to drop, one with unique queries is not."""- dags = list(DAG.objects.filter(team_id=team_id))- if len(dags) < 2:- return- dag_names = {dag.id: dag.name for dag in dags}- rows = (- Node.objects.filter(dag__in=dags)- .exclude(type=NodeType.TABLE)- .exclude(saved_query__deleted=True)- .values_list("dag_id", "saved_query_id")- )- dags_by_query: dict[object, set[object]] = defaultdict(set)- for dag_id, saved_query_id in rows:- if saved_query_id is not None:- dags_by_query[saved_query_id].add(dag_id)-- if not dags_by_query:- return- double_materialized = sum(1 for owning in dags_by_query.values() if len(owning) > 1)- unique_per_dag: dict[object, int] = defaultdict(int)- for owning in dags_by_query.values():- if len(owning) == 1:- unique_per_dag[next(iter(owning))] += 1-- self.stdout.write(f"\nTeam {team_id} cross-DAG duplication:")- self.stdout.write(f" {len(dags_by_query)} distinct saved queries across {len(dags)} DAGs")- self.stdout.write(f" in >1 DAG (double-materialized): {double_materialized}")- for dag in sorted(dags, key=lambda d: dag_names[d.id]):- unique = unique_per_dag.get(dag.id, 0)- note = "safe to drop" if unique == 0 else "unsafe to drop"- self.stdout.write(f" only in {dag_names[dag.id]} ({note}): {unique}")diff --git a/products/data_modeling/backend/management/commands/repoint_managed_view_readers.py b/products/data_modeling/backend/management/commands/repoint_managed_view_readers.pydeleted file mode 100644index eaac01f530be..000000000000--- a/products/data_modeling/backend/management/commands/repoint_managed_view_readers.py+++ /dev/null@@ -1,59 +0,0 @@-from collections import defaultdict-from typing import Any-from uuid import UUID--from django.core.management.base import BaseCommand, CommandParser-from django.db.models import QuerySet--import structlog--from products.data_modeling.backend.logic.saved_query_dag_sync import (- retire_stale_managed_view_copies,- stale_managed_view_copies,-)-from products.data_modeling.backend.models.node import Node--logger = structlog.get_logger(__name__)---class Command(BaseCommand):- help = (- "Move readers of stale managed-view copies onto the cross-DAG reference, then delete the copies. "- "Dry run unless --apply."- )-- def add_arguments(self, parser: CommandParser) -> None:- parser.add_argument("--team-id", type=int, default=None, help="Only clean up this team")- parser.add_argument("--apply", action="store_true", default=False, help="Repoint and delete instead of listing")-- def handle(self, *args: Any, **options: Any) -> None:- copy_ids_by_team = self._find_copies(options["team_id"])- copy_count = sum(len(copy_ids) for copy_ids in copy_ids_by_team.values())- self.stdout.write(f"{copy_count} stale managed view copies in {len(copy_ids_by_team)} team(s)")- if not options["apply"]:- return-- failed_team_ids: list[int] = []- for team_id, copy_ids in copy_ids_by_team.items():- try:- retire_stale_managed_view_copies(team_id, copy_ids)- except Exception:- logger.exception("Could not retire stale managed view copies", team_id=team_id)- failed_team_ids.append(team_id)- self.stdout.write(f"Done. {self._copies(options['team_id']).count()} copies remain")- if failed_team_ids:- self.stderr.write(f"Failed for team(s): {', '.join(str(team_id) for team_id in failed_team_ids)}")-- def _copies(self, team_id: int | None) -> QuerySet[Node]:- copies = stale_managed_view_copies()- if team_id is not None:- copies = copies.filter(team_id=team_id)- return copies-- def _find_copies(self, team_id: int | None) -> dict[int, list[UUID]]:- copies = self._copies(team_id)- copy_ids_by_team: dict[int, list[UUID]] = defaultdict(list)- for node_id, node_team_id, dag_id, name in copies.values_list("id", "team_id", "dag_id", "name").iterator():- self.stdout.write(f"team={node_team_id} dag={dag_id} node={node_id} name={name}")- copy_ids_by_team[node_team_id].append(node_id)- return copy_ids_by_teamdiff --git a/products/data_modeling/backend/models/node.py b/products/data_modeling/backend/models/node.pyindex 2983852af52d..5bf5d2c84c86 100644--- a/products/data_modeling/backend/models/node.py+++ b/products/data_modeling/backend/models/node.py@@ -25,7 +25,7 @@ SAVED_QUERY_NODE_TYPES = frozenset({NodeType.VIEW, NodeType.MAT_VIEW, NodeType.E_SAVED_QUERY_NODE_TYPE_VALUES = sorted(node_type.value for node_type in SAVED_QUERY_NODE_TYPES)-# properties["system"] marker set by consolidate_dags --adopt-unresolvable when a query's SQL+# properties["system"] marker set by adopt_saved_query_without_edges when a query's SQL# would not resolve and its node was created without edges. A successful sync clears it.DEGRADED_SYNC_KEY = "degraded_sync"diff --git a/products/data_modeling/backend/test/test_backfill_dag_schedule_type.py b/products/data_modeling/backend/test/test_backfill_dag_schedule_type.pydeleted file mode 100644index 2fb1702c751a..000000000000--- a/products/data_modeling/backend/test/test_backfill_dag_schedule_type.py+++ /dev/null@@ -1,55 +0,0 @@-from unittest import mock--from parameterized import parameterized-from temporalio.client import ScheduleListActionStartWorkflow-from temporalio.common import SearchAttributePair, TypedSearchAttributes--from posthog.temporal.common.search_attributes import POSTHOG_SCHEDULE_TYPE_KEY--from products.data_modeling.backend.management.commands.backfill_dag_schedule_type import _backfill_schedule_type-from products.data_modeling.backend.schedule import DATA_MODELING_EXECUTE_DAG_WORKFLOW---class TestBackfillScheduleType:- def _listing(self, schedule_id: str, workflow: str, schedule_type: str | None = None):- action = mock.Mock(spec=ScheduleListActionStartWorkflow, workflow=workflow)- attrs = TypedSearchAttributes(- [SearchAttributePair(POSTHOG_SCHEDULE_TYPE_KEY, schedule_type)] if schedule_type else []- )- return mock.Mock(id=schedule_id, schedule=mock.Mock(action=action), typed_search_attributes=attrs)-- def _temporal(self, listings):- async def fake_list_schedules(*args, **kwargs):- async def gen():- for listing in listings:- yield listing-- return gen()-- temporal = mock.Mock()- temporal.list_schedules = fake_list_schedules- temporal.get_schedule_handle = mock.Mock(return_value=mock.Mock(update=mock.AsyncMock()))- return temporal-- @parameterized.expand(- [- ("dry_run", True, 0, []),- ("real_run", False, 2, ["missing-1", "missing-2"]),- ]- )- def test_counts_stamped_vs_missing_and_updates_only_missing(self, _name, dry_run, expected_updated, updated_ids):- listings = [- self._listing("already-stamped", DATA_MODELING_EXECUTE_DAG_WORKFLOW, DATA_MODELING_EXECUTE_DAG_WORKFLOW),- self._listing("missing-1", DATA_MODELING_EXECUTE_DAG_WORKFLOW),- self._listing("missing-2", DATA_MODELING_EXECUTE_DAG_WORKFLOW),- self._listing("v1-schedule", "data-modeling-run"),- ]- temporal = self._temporal(listings)- with mock.patch(- "products.data_modeling.backend.management.commands.backfill_dag_schedule_type.async_connect",- new=mock.AsyncMock(return_value=temporal),- ):- found, already_stamped, updated, failed = _backfill_schedule_type(dry_run)-- assert (found, already_stamped, updated, failed) == (3, 1, expected_updated, 0)- assert [call.args[0] for call in temporal.get_schedule_handle.call_args_list] == updated_idsdiff --git a/products/data_modeling/backend/test/test_backfill_saved_query_nodes.py b/products/data_modeling/backend/test/test_backfill_saved_query_nodes.pydeleted file mode 100644index 8cae8d080419..000000000000--- a/products/data_modeling/backend/test/test_backfill_saved_query_nodes.py+++ /dev/null@@ -1,54 +0,0 @@-from datetime import timedelta--from posthog.test.base import BaseTest-from unittest.mock import patch--from django.core.management import call_command--from products.data_modeling.backend.logic.node_frequency import get_declared_target-from products.data_modeling.backend.models.dag import DAG-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery-from products.data_modeling.backend.models.edge import Edge-from products.data_modeling.backend.models.node import LineageIssueKind, Node--COMMAND = "products.data_modeling.backend.management.commands.backfill_saved_query_nodes"---class TestBackfillSavedQueryNodes(BaseTest):- def _view(- self, name: str, sql: str, is_materialized: bool = False, interval: timedelta | None = None- ) -> DataWarehouseSavedQuery:- return DataWarehouseSavedQuery.objects.create(- team=self.team,- name=name,- query={"kind": "HogQLQuery", "query": sql},- is_materialized=is_materialized,- sync_frequency_interval=interval,- )-- def test_gives_every_nodeless_view_a_node_only_when_applied(self) -> None:- downstream = self._view("downstream_view", "SELECT event FROM upstream_view")- upstream = self._view("upstream_view", "SELECT event FROM events")- broken = self._view("broken_view", "SELECT 1 FROM no_such_table")- materialized = self._view("materialized_view", "SELECT event FROM events", is_materialized=True)- legacy_cadence = self._view(- "legacy_cadence_view", "SELECT event FROM events", is_materialized=True, interval=timedelta(minutes=5)- )-- with patch(f"{COMMAND}.reconcile_dag_schedules") as reconcile:- call_command("backfill_saved_query_nodes")- self.assertFalse(Node.objects.filter(team=self.team).exists())-- call_command("backfill_saved_query_nodes", "--apply")-- reconcile.assert_called_once_with(DAG.get_or_create_default(self.team), require_tiered=True)- upstream_node = Node.objects.get(saved_query=upstream)- downstream_node = Node.objects.get(saved_query=downstream)- self.assertTrue(Edge.objects.filter(source=upstream_node, target=downstream_node).exists())- broken_node = Node.objects.get(saved_query=broken)- self.assertFalse(Edge.objects.filter(target=broken_node).exists())- assert broken_node.lineage_issue is not None- self.assertEqual(broken_node.lineage_issue["kind"], LineageIssueKind.SYNC_FAILED)- self.assertEqual(get_declared_target(Node.objects.get(saved_query=materialized)), timedelta(days=1))- self.assertEqual(get_declared_target(Node.objects.get(saved_query=legacy_cadence)), timedelta(minutes=15))- self.assertIsNone(get_declared_target(upstream_node))diff --git a/products/data_modeling/backend/test/test_consolidate_dags.py b/products/data_modeling/backend/test/test_consolidate_dags.pydeleted file mode 100644index adcda6127698..000000000000--- a/products/data_modeling/backend/test/test_consolidate_dags.py+++ /dev/null@@ -1,777 +0,0 @@-import uuid-from contextlib import contextmanager-from datetime import timedelta-from io import StringIO-from types import SimpleNamespace--import pytest-from posthog.test.base import BaseTest-from unittest import mock--from django.core.management import call_command-from django.core.management.base import CommandError--from parameterized import parameterized-from temporalio.client import ScheduleListActionStartWorkflow--from posthog.models.team import Team--from products.data_modeling.backend.logic.cohort_scheduling import tier_schedule_id-from products.data_modeling.backend.logic.node_frequency import get_declared_target, set_declared_target-from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag-from products.data_modeling.backend.logic.schedule_reconcile import DagScheduleTeardown-from products.data_modeling.backend.models.dag import DAG, DEFAULT_DAG_NAME, REVENUE_ANALYTICS_DAG_NAME-from products.data_modeling.backend.models.data_modeling_job import DataModelingJob-from products.data_modeling.backend.models.datawarehouse_managed_viewset import DataWarehouseManagedViewSet-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery-from products.data_modeling.backend.models.edge import Edge-from products.data_modeling.backend.models.node import Node, NodeType-from products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKind--M15 = timedelta(minutes=15)-H6 = timedelta(hours=6)-H12 = timedelta(hours=12)--COMMAND = "products.data_modeling.backend.management.commands.consolidate_dags"-RECONCILE = "products.data_modeling.backend.logic.schedule_reconcile"---def _listing_client(schedules_by_dag):- # query-aware fake: schedule listings are per-DAG (PostHogDagId search attribute), and the- # command lists the target, each source, and the reconcile path — they must not see each- # other's schedules- def _listing(schedule_id):- action = mock.Mock(spec=ScheduleListActionStartWorkflow, workflow="data-modeling-execute-dag")- return mock.Mock(id=schedule_id, schedule=mock.Mock(action=action))-- async def fake_list_schedules(*_args, query="", **_kwargs):- dag_id = query.split("'")[1] if "'" in query else ""-- async def gen():- for schedule_id in schedules_by_dag.get(dag_id, []):- yield _listing(schedule_id)-- return gen()-- client = mock.Mock()- client.list_schedules = fake_list_schedules- return client---@contextmanager-def _temporal_boundary(schedules_by_dag=None, v2_teardown_fails=False):- v2_delete = mock.Mock(return_value=DagScheduleTeardown(ok=not v2_teardown_fails, deleted=()))- with (- mock.patch(f"{COMMAND}.sync_connect"),- mock.patch(f"{COMMAND}.delete_dag_schedules", v2_delete),- mock.patch(f"{RECONCILE}.delete_schedule") as v1_delete,- mock.patch(- f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=_listing_client(schedules_by_dag or {}))- ),- mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()) as tier_create,- mock.patch(f"{RECONCILE}.a_update_schedule", new=mock.AsyncMock()) as tier_update,- mock.patch(f"{RECONCILE}.a_delete_schedule", new=mock.AsyncMock()),- ):- yield SimpleNamespace(- v1_delete=v1_delete, v2_delete=v2_delete, tier_create=tier_create, tier_update=tier_update- )---class TestConsolidateDags(BaseTest):- def _query(self, name: str, sql: str, interval: timedelta | None = None) -> DataWarehouseSavedQuery:- return DataWarehouseSavedQuery.objects.create(- name=name, team=self.team, query={"query": sql, "kind": "HogQLQuery"}, sync_frequency_interval=interval- )-- def _node(self, dag: DAG, saved_query: DataWarehouseSavedQuery) -> Node:- return Node.objects.create(team=self.team, dag=dag, saved_query=saved_query, type=NodeType.MAT_VIEW)-- def _run(self, *args: str, apply: bool = False) -> str:- out = StringIO()- err = StringIO()- flags = ["--apply"] if apply else []- call_command("consolidate_dags", "--team-id", str(self.team.pk), *flags, *args, stdout=out, stderr=err)- return out.getvalue() + err.getvalue()-- def test_end_to_end_shared_dropped_unique_moved_with_rebuilt_edges(self):- # The moved query depends on the dropped one, so its edges must rebuild against the- # target's existing copy; the source DAG and both its nodes must be gone afterwards.- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- shared = self._query("shared_mv", "SELECT 1")- target_shared_node = self._node(target, shared)- self._node(source, shared)- moved = self._query("only_in_source", "SELECT * FROM shared_mv")- self._node(source, moved)-- with _temporal_boundary():- output = self._run(apply=True)-- self.assertFalse(DAG.objects.filter(id=source.id).exists())- moved_node = Node.objects.get(dag=target, saved_query=moved)- self.assertTrue(Edge.objects.filter(dag=target, source=target_shared_node, target=moved_node).exists())- self.assertEqual(Node.objects.filter(saved_query=shared).count(), 1)- self.assertEqual(Node.objects.filter(saved_query=moved).count(), 1)- self.assertIn("moved only_in_source", output)- self.assertIn("dropped duplicate shared_mv", output)- self.assertIn("deleted source DAG", output)-- def test_moves_sync_in_dependency_order(self):- # dep_child depends on dep_parent, both only in the source: syncing the child first would- # fail dependency resolution in the target (Node.DoesNotExist) and keep the source DAG.- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- parent = self._query("dep_parent", "SELECT * FROM events")- child = self._query("dep_child", "SELECT * FROM dep_parent")- parent_node = self._node(source, parent)- child_node = self._node(source, child)- Edge.objects.create(team=self.team, dag=source, source=parent_node, target=child_node)-- with _temporal_boundary():- self._run(apply=True)-- self.assertFalse(DAG.objects.filter(id=source.id).exists())- new_parent = Node.objects.get(dag=target, saved_query=parent)- new_child = Node.objects.get(dag=target, saved_query=child)- self.assertTrue(Edge.objects.filter(dag=target, source=new_parent, target=new_child).exists())- events = Node.objects.get(dag=target, name="events", type=NodeType.TABLE)- self.assertTrue(Edge.objects.filter(dag=target, source=events, target=new_parent).exists())-- @parameterized.expand(- [- ("tiered", "tier", True, True),- ("legacy_v2", "legacy", True, False),- ("v1_only", "none", False, False),- ]- )- def test_target_schedule_mode_controls_teardown(self, _name, existing_kind, expect_sweep, expect_reconcile):- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- shared = self._query("shared_mv", "SELECT 1", interval=H6)- self._node(target, shared)- self._node(source, shared)- moved = self._query("only_in_source", "SELECT * FROM events", interval=H6)- self._node(source, moved)-- tier_id = tier_schedule_id(str(target.id), H6)- schedules_by_dag = {- "tier": {str(target.id): [tier_id]},- "legacy": {str(target.id): [str(target.id)]},- "none": {},- }[existing_kind]-- with _temporal_boundary(schedules_by_dag=schedules_by_dag) as mocks:- self._run(apply=True)-- self.assertFalse(DAG.objects.filter(id=source.id).exists())- moved.refresh_from_db()- shared.refresh_from_db()- moved_node = Node.objects.get(dag=target, saved_query=moved)- if expect_sweep:- swept = {call.kwargs["schedule_id"] for call in mocks.v1_delete.call_args_list}- self.assertEqual(swept, {str(moved.id), str(shared.id)})- self.assertIsNone(moved.sync_frequency_interval)- self.assertIsNone(shared.sync_frequency_interval)- # cadence intent survives the nulled interval as a declared node target- self.assertEqual(get_declared_target(moved_node), H6)- else:- mocks.v1_delete.assert_not_called()- self.assertEqual(moved.sync_frequency_interval, H6)- self.assertEqual(shared.sync_frequency_interval, H6)- self.assertIsNone(get_declared_target(moved_node))- if expect_reconcile:- mocks.tier_update.assert_called_once()- self.assertEqual(mocks.tier_update.call_args.kwargs["id"], tier_id)- mocks.tier_create.assert_not_called()- else:- mocks.tier_update.assert_not_called()- mocks.tier_create.assert_not_called()-- def test_refuses_v2_scheduled_source_when_target_is_v1_only(self):- # moving these queries would strand them: their execute-dag schedules die with the source- # DAG but a v1-only target has no schedule to hand coverage to- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- moved = self._query("only_in_source", "SELECT 1", interval=H6)- self._node(source, moved)-- with _temporal_boundary(schedules_by_dag={str(source.id): [str(source.id)]}) as mocks:- with self.assertRaisesRegex(CommandError, "v2-scheduled"):- self._run(apply=True)-- self.assertTrue(DAG.objects.filter(id=source.id).exists())- self.assertFalse(Node.objects.filter(dag=target, saved_query=moved).exists())- moved.refresh_from_db()- self.assertEqual(moved.sync_frequency_interval, H6)- mocks.v1_delete.assert_not_called()- mocks.v2_delete.assert_not_called()-- def test_rerun_finalizes_target_after_crashed_run(self):- # a prior run that crashed after deleting its source DAGs leaves a moved query on the- # target with its interval intact, no declared target, and a live v1 schedule; a plain- # re-run must finish seed/sweep/null/reconcile from the target's own state- target = DAG.get_or_create_default(self.team)- stranded = self._query("stranded_mv", "SELECT 1", interval=H6)- node = self._node(target, stranded)- tier_id = tier_schedule_id(str(target.id), H6)-- with _temporal_boundary(schedules_by_dag={str(target.id): [tier_id]}) as mocks:- self._run(apply=True)-- node.refresh_from_db()- stranded.refresh_from_db()- self.assertEqual(get_declared_target(node), H6)- self.assertIsNone(stranded.sync_frequency_interval)- swept = {call.kwargs["schedule_id"] for call in mocks.v1_delete.call_args_list}- self.assertEqual(swept, {str(stranded.id)})- mocks.tier_update.assert_called_once()- self.assertEqual(mocks.tier_update.call_args.kwargs["id"], tier_id)-- def test_source_dag_interval_carries_as_declared_target(self):- # Legacy-migrated source shape: query intervals nulled, cadence lives only on- # dag.sync_frequency_interval. The finalize seeds from the *target* DAG, whose interval is- # NULL here, so unless the source cadence is captured at plan time the moved query arrives- # with no effective cadence and reconcile drops it from every tier.- target = DAG.get_or_create_default(self.team)- target.sync_frequency_interval = None- target.save()- source = DAG.objects.create(team=self.team, name="posthog_team", sync_frequency_interval=H6)- moved = self._query("only_in_source", "SELECT 1")- self._node(source, moved)- tier_id = tier_schedule_id(str(target.id), H6)-- with _temporal_boundary(schedules_by_dag={str(target.id): [tier_id], str(source.id): [str(source.id)]}):- self._run(apply=True)-- self.assertFalse(DAG.objects.filter(id=source.id).exists())- moved_node = Node.objects.get(dag=target, saved_query=moved)- self.assertEqual(get_declared_target(moved_node), H6)-- def test_finalize_redetects_target_mode_after_mid_run_conversion(self):- # The pre-flight check sees a v1-only target; a concurrent reconcile converts it to tiers- # before the finalize. Acting on the stale snapshot would skip the seed/sweep/reconcile and- # leave the moved query double-scheduled (v1 + tier).- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- moved = self._query("only_in_source", "SELECT 1", interval=H6)- self._node(source, moved)- tier_id = tier_schedule_id(str(target.id), H6)-- target_listings = iter([set(), {tier_id}])-- def fake_listing(dag_id):- if dag_id == str(target.id):- return next(target_listings)- return set()-- with (- _temporal_boundary(schedules_by_dag={str(target.id): [tier_id]}) as mocks,- mock.patch(f"{COMMAND}.list_existing_schedule_ids", side_effect=fake_listing),- ):- self._run(apply=True)-- moved.refresh_from_db()- self.assertIsNone(moved.sync_frequency_interval)- swept = {call.kwargs["schedule_id"] for call in mocks.v1_delete.call_args_list}- self.assertEqual(swept, {str(moved.id)})- mocks.tier_update.assert_called_once()- self.assertEqual(get_declared_target(Node.objects.get(dag=target, saved_query=moved)), H6)-- def test_failed_move_keeps_source_dag_and_raises(self):- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- good = self._query("good_view", "SELECT 1")- bad = self._query("bad_view", "SELECT * FROM nonexistent_table_xyz")- self._node(source, good)- self._node(source, bad)-- with _temporal_boundary() as mocks:- with self.assertRaisesRegex(CommandError, "incomplete"):- self._run(apply=True)-- # the DAG survives so the failed query's only node is not destroyed; the successful move- # remains in the target as a duplicate a re-run classifies as DROP- self.assertTrue(DAG.objects.filter(id=source.id).exists())- self.assertTrue(Node.objects.filter(dag=source, saved_query=bad).exists())- self.assertTrue(Node.objects.filter(dag=target, saved_query=good).exists())- mocks.v2_delete.assert_not_called()-- def test_adopt_unresolvable_lands_broken_query_edge_less_and_deletes_source_dag(self):- # One broken query must not hold the team's consolidation hostage: with the flag, the- # unresolvable query is adopted into the target with no edges and a marker, cadence intent- # intact, while healthy queries move normally and the source DAG is deleted.- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- good = self._query("good_view", "SELECT * FROM events")- bad = self._query("bad_view", "SELECT * FROM nonexistent_table_xyz")- self._node(source, good)- set_declared_target(self._node(source, bad), M15)-- with _temporal_boundary():- output = self._run("--adopt-unresolvable", apply=True)-- self.assertFalse(DAG.objects.filter(id=source.id).exists())- adopted = Node.objects.get(dag=target, saved_query=bad)- self.assertFalse(Edge.objects.filter(dag=target, target=adopted).exists())- self.assertIn("nonexistent_table_xyz", adopted.properties["system"]["degraded_sync"]["error"])- self.assertEqual(get_declared_target(adopted), M15)- good_node = Node.objects.get(dag=target, saved_query=good)- events = Node.objects.get(dag=target, name="events", type=NodeType.TABLE)- self.assertTrue(Edge.objects.filter(dag=target, source=events, target=good_node).exists())- self.assertIn("adopted bad_view without edges", output)-- def test_adopted_node_unblocks_its_dependents_moves(self):- # The reason the flag exists: one unresolvable query must not cascade into N failed- # moves. blocked_view depends on a saved query that has no DAG node (an orphan) → it- # resolves in HogQL but fails Node.DoesNotExist and is adopted edge-less → its dependent- # resolves the adopted node in the target and moves with a real edge.- target = DAG.get_or_create_default(self.team)- self._query("orphan_parent", "SELECT 1")- source = DAG.objects.create(team=self.team, name="posthog_team")- blocked = self._query("blocked_view", "SELECT * FROM orphan_parent")- dependent = self._query("dependent_view", "SELECT * FROM blocked_view")- blocked_node = self._node(source, blocked)- dependent_node = self._node(source, dependent)- Edge.objects.create(team=self.team, dag=source, source=blocked_node, target=dependent_node)-- with _temporal_boundary():- output = self._run("--adopt-unresolvable", apply=True)-- self.assertFalse(DAG.objects.filter(id=source.id).exists())- adopted = Node.objects.get(dag=target, saved_query=blocked)- self.assertFalse(Edge.objects.filter(dag=target, target=adopted).exists())- self.assertIn("degraded_sync", adopted.properties["system"])- moved = Node.objects.get(dag=target, saved_query=dependent)- self.assertTrue(Edge.objects.filter(dag=target, source=adopted, target=moved).exists())- self.assertIn("adopted blocked_view without edges", output)- self.assertIn("moved dependent_view", output)-- def test_adopt_unresolvable_does_not_adopt_non_resolution_failures(self):- # The flag exists for SQL that cannot resolve. A sync that fails for operational reasons- # (DB blip, programming error) must stay a failed move — adopting it would delete the- # source DAG and report success on a query a plain retry would have moved intact.- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- moved = self._query("only_in_source", "SELECT 1")- self._node(source, moved)-- with _temporal_boundary():- with mock.patch(f"{COMMAND}.sync_saved_query_to_dag", side_effect=RuntimeError("db down")):- with self.assertRaisesRegex(CommandError, "incomplete"):- self._run("--adopt-unresolvable", apply=True)-- self.assertTrue(DAG.objects.filter(id=source.id).exists())- self.assertFalse(Node.objects.filter(dag=target, saved_query=moved).exists())-- def test_adoption_failure_rolls_back_the_node(self):- # If the marker save fails after get_or_create committed the node, an unmarked edge-less- # node would survive; the re-run would then classify the query as DROP instead of- # retrying the move, stranding it without the marker forever. Create + mark must commit- # or roll back together so the re-run sees a clean MOVE to retry.- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- bad = self._query("bad_view", "SELECT * FROM nonexistent_table_xyz")- self._node(source, bad)-- real_save = Node.save-- def failing_marker_save(node, *args, **kwargs):- if kwargs.get("update_fields") == ["properties"]:- raise RuntimeError("connection lost")- return real_save(node, *args, **kwargs)-- with _temporal_boundary():- with mock.patch.object(Node, "save", autospec=True, side_effect=failing_marker_save):- with self.assertRaisesRegex(CommandError, "incomplete"):- self._run("--adopt-unresolvable", apply=True)-- self.assertTrue(DAG.objects.filter(id=source.id).exists())- self.assertFalse(Node.objects.filter(dag=target, saved_query=bad).exists())-- def test_degraded_sync_marker_cleared_when_query_resolves_again(self):- # A stale marker after the customer fixes their SQL would make the post-fix re-sync sweep- # (and any dashboard keyed on the marker) report a healthy node as degraded forever.- target = DAG.get_or_create_default(self.team)- fixed = self._query("fixed_view", "SELECT * FROM events")- node = Node.objects.create(- team=self.team,- dag=target,- saved_query=fixed,- type=NodeType.VIEW,- properties={"system": {"degraded_sync": {"error": "Unknown table", "at": "2026-07-30T00:00:00+00:00"}}},- )-- sync_saved_query_to_dag(fixed, dag=target, reconcile=False)-- node.refresh_from_db()- self.assertNotIn("system", node.properties)- events = Node.objects.get(dag=target, name="events", type=NodeType.TABLE)- self.assertTrue(Edge.objects.filter(dag=target, source=events, target=node).exists())-- def test_schedule_teardown_failure_keeps_source_dag(self):- # deleting the DAG row before its execute-dag schedules are gone would orphan the- # schedules forever (the row is the only pointer to them)- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- moved = self._query("only_in_source", "SELECT 1")- self._node(source, moved)- schedules_by_dag = {str(target.id): [str(target.id)], str(source.id): [str(source.id)]}-- with _temporal_boundary(schedules_by_dag=schedules_by_dag, v2_teardown_fails=True):- with self.assertRaisesRegex(CommandError, "incomplete"):- self._run(apply=True)-- self.assertTrue(DAG.objects.filter(id=source.id).exists())- self.assertTrue(Node.objects.filter(dag=target, saved_query=moved).exists())-- def test_dry_run_by_default_mutates_nothing(self):- source = DAG.objects.create(team=self.team, name="posthog_team")- target = DAG.get_or_create_default(self.team)- shared = self._query("shared_mv", "SELECT 1", interval=H6)- self._node(target, shared)- self._node(source, shared)- moved = self._query("only_in_source", "SELECT 1", interval=H6)- self._node(source, moved)-- with _temporal_boundary() as mocks:- output = self._run()-- self.assertIn("move 1, drop 1", output)- self.assertIn("dry run", output)- self.assertTrue(DAG.objects.filter(id=source.id).exists())- self.assertFalse(Node.objects.filter(dag=target, saved_query=moved).exists())- moved.refresh_from_db()- self.assertEqual(moved.sync_frequency_interval, H6)- mocks.v1_delete.assert_not_called()- mocks.v2_delete.assert_not_called()-- def test_dry_run_reports_edge_impact(self):- # An edge touching a moved node must be re-pointed to the target; an edge whose only- # involved node is dropped is redundant against the target's existing edge.- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- shared = self._query("shared_mv", "SELECT 1")- self._node(target, shared)- dropped_node = self._node(source, shared)- moved_node = self._node(source, self._query("only_in_source", "SELECT 1"))- events = Node.objects.create(team=self.team, dag=source, name="events", type=NodeType.TABLE)- Edge.objects.create(team=self.team, dag=source, source=events, target=moved_node)- Edge.objects.create(team=self.team, dag=source, source=events, target=dropped_node)-- output = self._run()-- self.assertIn("edges to re-point (touch a moved node): 1", output)- self.assertIn("edges to drop as redundant (only dropped nodes): 1", output)- self.assertIn("edges to re-point: 1, edges to drop: 1", output)-- def test_duplicate_is_kept_when_the_claiming_dag_failed_to_move_it(self):- # dag_a claims the shared query for MOVE and dag_b's copy is planned as a DROP. If dag_a's- # move fails, dag_b's node is the last place that query's declared target exists — dropping- # it and deleting dag_b would destroy the intent with only a "dropped duplicate" line.- DAG.get_or_create_default(self.team)- dag_a = DAG.objects.create(team=self.team, name="dag_a")- dag_b = DAG.objects.create(team=self.team, name="dag_b")- shared = self._query("shared_mv", "SELECT * FROM nonexistent_table_xyz")- self._node(dag_a, shared)- self._node(dag_b, shared)-- with _temporal_boundary():- with self.assertRaisesRegex(CommandError, "incomplete"):- self._run(apply=True)-- # dag_a could not move it, so dag_b's copy is the last node the query has anywhere- self.assertTrue(DAG.objects.filter(id=dag_b.id).exists())- self.assertTrue(Node.objects.filter(dag=dag_b, saved_query=shared).exists())-- def test_dag_default_cadence_seeds_a_moved_node_with_no_interval(self):- # A query only ever scheduled by its DAG's default cadence has no interval of its own. The- # finalize nulls every interval and then reconciles, so with no DAG-level fallback the moved- # node arrives with nothing declared, gets no effective cadence, and is dropped from every- # tier — it stops materializing with nothing left in Postgres or Temporal to show why.- target = DAG.get_or_create_default(self.team)- target.sync_frequency_interval = H6- target.save()- source = DAG.objects.create(team=self.team, name="posthog_team")- moved = self._query("only_in_source", "SELECT 1") # no interval of its own- self._node(source, moved)-- tier_id = tier_schedule_id(str(target.id), H6)- with _temporal_boundary(schedules_by_dag={str(target.id): [tier_id]}):- self._run(apply=True)-- moved_node = Node.objects.get(dag=target, saved_query=moved)- self.assertEqual(get_declared_target(moved_node), H6)-- def test_verbose_lists_query_names(self):- # --verbose is how an operator confirms *which* queries move before committing to --apply- # on live data, so the per-name listing has to actually appear behind the flag.- DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- self._node(source, self._query("only_in_source", "SELECT 1"))-- self.assertNotIn("only_in_source", self._run())- self.assertIn("move (in dependency order): only_in_source", self._run("--verbose"))-- def test_conflicting_freshness_targets_flagged(self):- # Same query in two DAGs with different declared cadences: only one node survives the merge,- # so the dry run must surface both copies for a human to pick the winner.- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- shared = self._query("conflicted", "SELECT 1")- set_declared_target(self._node(target, shared), H6)- set_declared_target(self._node(source, shared), M15)-- output = self._run()-- self.assertIn("freshness-target conflicts: 1", output)- self.assertIn("⚠ conflicted:", output)- self.assertIn("6hour", output)- self.assertIn("15min", output)-- @parameterized.expand([("both_agree", H6, H6), ("one_unset", H6, None), ("both_unset", None, None)])- def test_non_conflicting_freshness_targets_not_flagged(self, _name, target_a, target_b):- # Unset means "no opinion", so a lone declared target wins cleanly — not a conflict.- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- shared = self._query("agreeing", "SELECT 1")- a = self._node(target, shared)- b = self._node(source, shared)- if target_a is not None:- set_declared_target(a, target_a)- if target_b is not None:- set_declared_target(b, target_b)-- self.assertIn("freshness-target conflicts: 0", self._run())-- @parameterized.expand(- [- ("default_preferred", "default"),- ("largest_without_default", "largest"),- ("explicit_wins", "explicit"),- ]- )- def test_target_selection(self, _name, kind):- if kind == "largest":- small = DAG.objects.create(team=self.team, name="small")- large = DAG.objects.create(team=self.team, name="large")- self._node(small, self._query("a", "SELECT 1"))- self._node(large, self._query("b", "SELECT 1"))- self._node(large, self._query("c", "SELECT 1"))-- output = self._run()-- self.assertIn(f"Merge target: large ({large.id})", output)- self.assertIn("largest DAG by node count", output)- return-- default = DAG.get_or_create_default(self.team)- other = DAG.objects.create(team=self.team, name="posthog_team")- self._node(other, self._query("a", "SELECT 1"))-- if kind == "explicit":- output = self._run("--target-dag-id", str(other.id))- self.assertIn(f"Merge target: posthog_team ({other.id})", output)- self.assertIn("chosen via --target-dag-id", output)- else:- output = self._run()- self.assertIn(f"Merge target: Default ({default.id})", output)- self.assertIn("canonical 'Default' DAG", output)-- @parameterized.expand(- [- ("default_dag_exists", DEFAULT_DAG_NAME),- ("only_a_differently_named_dag", "posthog_team"),- ("no_user_dag_yet", None),- ]- )- def test_managed_dag_folds_into_the_default_dag(self, _name, user_dag_name):- managed = DAG.get_or_create_revenue_analytics(self.team)- managed_query = self._query("stripe.prod.mrr_revenue_view", "SELECT 1")- managed_node = self._node(managed, managed_query)- set_declared_target(managed_node, H12)- if user_dag_name is not None:- default = DAG.objects.create(team=self.team, name=user_dag_name)- reader = self._node(default, self._query("reads_mrr", "SELECT 1"))- proxy = Node.objects.create(- team=self.team,- dag=default,- name=managed_query.name,- type=NodeType.TABLE,- properties={"origin": "cross_dag_view", "saved_query_id": str(managed_query.id)},- )- Edge.objects.create(team=self.team, dag=default, source=proxy, target=reader)-- with _temporal_boundary() as mocks:- self._run(apply=True)-- target = DAG.objects.get(team=self.team)- self.assertEqual(target.name, DEFAULT_DAG_NAME)- node = Node.objects.get(saved_query=managed_query)- self.assertEqual((node.id, node.dag_id), (managed_node.id, target.id))- self.assertEqual(get_declared_target(node), H12)- self.assertEqual(mocks.v2_delete.call_args.args, (str(managed.id),))- self.assertEqual(mocks.tier_create.call_args.kwargs["id"], tier_schedule_id(str(target.id), H12))- if user_dag_name is not None:- self.assertEqual(list(Edge.objects.filter(target=reader).values_list("source", flat=True)), [node.id])- self.assertFalse(Node.objects.filter(properties__has_key="saved_query_id").exists())-- @parameterized.expand([("teardown_succeeds", False), ("teardown_fails", True)])- def test_teams_with_managed_dag_halts_at_the_first_failing_team(self, _name, teardown_fails):- other_team = Team.objects.create(organization=self.organization)- managed_dags = []- for team in sorted([self.team, other_team], key=lambda team: team.pk):- managed = DAG.objects.create(team=team, name=REVENUE_ANALYTICS_DAG_NAME)- managed_query = DataWarehouseSavedQuery.objects.create(- name="mrr_revenue_view", team=team, query={"query": "SELECT 1", "kind": "HogQLQuery"}- )- Node.objects.create(team=team, dag=managed, saved_query=managed_query, type=NodeType.MAT_VIEW)- managed_dags.append(managed)-- with _temporal_boundary(v2_teardown_fails=teardown_fails):- if teardown_fails:- with self.assertRaisesRegex(CommandError, "halted at team"):- call_command("consolidate_dags", "--teams-with-managed-dag", "--apply", stdout=StringIO())- else:- call_command("consolidate_dags", "--teams-with-managed-dag", "--apply", stdout=StringIO())-- remaining = set(DAG.objects.filter(name=REVENUE_ANALYTICS_DAG_NAME).values_list("id", flat=True))- self.assertEqual(remaining, {dag.id for dag in managed_dags} if teardown_fails else set())-- def test_a_running_materialization_keeps_the_managed_dag_and_its_schedules(self):- DAG.get_or_create_default(self.team)- managed = DAG.get_or_create_revenue_analytics(self.team)- managed_query = self._query("stripe.prod.mrr_revenue_view", "SELECT 1")- managed_node = self._node(managed, managed_query)- DataModelingJob.objects.create(team=self.team, saved_query=managed_query, status=DataModelingJob.Status.RUNNING)-- with _temporal_boundary() as mocks:- with self.assertRaisesRegex(CommandError, "incomplete"):- self._run(apply=True)-- mocks.v2_delete.assert_not_called()- self.assertEqual(Node.objects.get(saved_query=managed_query).dag_id, managed_node.dag_id)-- def test_a_managed_view_that_fails_to_resolve_stays_in_the_managed_dag_with_its_edges(self):- DAG.get_or_create_default(self.team)- managed = DAG.get_or_create_revenue_analytics(self.team)- managed_query = self._query("stripe.prod.mrr_revenue_view", "SELECT * FROM nonexistent_table_xyz")- managed_node = self._node(managed, managed_query)- parent = Node.objects.create(team=self.team, dag=managed, name="events", type=NodeType.TABLE)- Edge.objects.create(team=self.team, dag=managed, source=parent, target=managed_node)-- with _temporal_boundary():- with self.assertRaisesRegex(CommandError, "incomplete"):- self._run(apply=True)-- self.assertTrue(DAG.objects.filter(id=managed.id).exists())- self.assertEqual(Node.objects.get(saved_query=managed_query).dag_id, managed.id)- self.assertTrue(Edge.objects.filter(dag=managed, source=parent, target=managed_node).exists())-- def test_folding_into_an_unscheduled_dag_seeds_targets_from_intervals(self):- managed = DAG.get_or_create_revenue_analytics(self.team)- managed_query = self._query("stripe.prod.mrr_revenue_view", "SELECT 1", interval=H12)- self._node(managed, managed_query)-- with _temporal_boundary() as mocks:- self._run(apply=True)-- managed_query.refresh_from_db()- self.assertIsNone(managed_query.sync_frequency_interval)- self.assertEqual(get_declared_target(Node.objects.get(saved_query=managed_query)), H12)- self.assertEqual(- mocks.tier_create.call_args.kwargs["id"],- tier_schedule_id(str(DAG.objects.get(team=self.team).id), H12),- )-- def test_a_managed_view_lands_from_the_managed_dag_not_from_a_user_dag_copy(self):- target = DAG.get_or_create_default(self.team)- managed = DAG.get_or_create_revenue_analytics(self.team)- viewset = DataWarehouseManagedViewSet.objects.create(- team=self.team, kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS- )- managed_query = self._query("stripe.prod.mrr_revenue_view", "SELECT 1")- unplaced_managed_query = self._query("stripe.prod.charge_revenue_view", "SELECT 1")- for query in (managed_query, unplaced_managed_query):- query.managed_viewset = viewset- query.save(update_fields=["managed_viewset"])- set_declared_target(self._node(managed, managed_query), H12)- set_declared_target(self._node(target, managed_query), timedelta(days=1))- source = DAG.objects.create(team=self.team, name="posthog_team")- self._node(source, managed_query)- self._node(source, unplaced_managed_query)- moved = self._query("only_in_source", "SELECT 1")- self._node(source, moved)-- with _temporal_boundary():- self._run(apply=True)-- self.assertEqual(list(DAG.objects.filter(team=self.team).values_list("id", flat=True)), [target.id])- managed_node = Node.objects.get(saved_query=managed_query)- self.assertEqual(managed_node.dag_id, target.id)- self.assertEqual(get_declared_target(managed_node), H12)- self.assertTrue(Node.objects.filter(dag=target, saved_query=moved).exists())- self.assertTrue(Node.objects.filter(dag=target, saved_query=unplaced_managed_query).exists())-- @parameterized.expand(- [- ("managed_target", "managed"),- ("unknown_target", "unknown"),- ("not_a_uuid", "garbage"),- ]- )- def test_bad_target_dag_id_errors_before_any_change(self, _name, kind):- DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- self._node(source, self._query("only_in_source", "SELECT 1"))- managed = DAG.get_or_create_revenue_analytics(self.team)- target_id = {"managed": str(managed.id), "unknown": str(uuid.uuid4()), "garbage": "not-a-uuid"}[kind]-- with _temporal_boundary():- with self.assertRaises(CommandError):- self._run("--target-dag-id", target_id, apply=True)-- self.assertTrue(DAG.objects.filter(id=source.id).exists())-- @parameterized.expand([("two_sources", 2), ("three_sources", 3)])- def test_query_in_several_source_dags_lands_exactly_once(self, _name: str, source_count: int):- # `saved_query_unique_within_team_dag` means the query can only land once: the first source- # moves it, every later one must drop. Three sources exercises a drop against a target that- # already holds the node, which two sources never reaches.- target = DAG.get_or_create_default(self.team)- shared = self._query("shared_everywhere", "SELECT 1")- sources = [DAG.objects.create(team=self.team, name=f"dag_{i}") for i in range(source_count)]- for source in sources:- self._node(source, shared)-- with _temporal_boundary():- output = self._run(apply=True)-- self.assertFalse(DAG.objects.filter(id__in=[dag.id for dag in sources]).exists())- self.assertEqual(Node.objects.filter(saved_query=shared).count(), 1)- self.assertTrue(Node.objects.filter(dag=target, saved_query=shared).exists())- self.assertIn(f"moved: 1, dropped: {source_count - 1}", output)-- @parameterized.expand(- [- ("moved_node_inherits", "move", None, M15),- ("dropped_copies_onto_unset_target", "drop", None, M15),- ("target_declared_wins", "drop", H6, H6),- ]- )- def test_declared_target_carried_from_source(self, _name, kind, target_declared, expected):- target = DAG.get_or_create_default(self.team)- source = DAG.objects.create(team=self.team, name="posthog_team")- saved_query = self._query("carried", "SELECT 1")- if kind == "drop":- target_node = self._node(target, saved_query)- if target_declared is not None:- set_declared_target(target_node, target_declared)- source_node = self._node(source, saved_query)- set_declared_target(source_node, M15)-- with _temporal_boundary():- self._run(apply=True)-- node = Node.objects.get(dag=target, saved_query=saved_query)- self.assertEqual(get_declared_target(node), expected)diff --git a/products/data_modeling/backend/test/test_preview_freshness_schedules.py b/products/data_modeling/backend/test/test_preview_freshness_schedules.pyindex e4c979a9022d..df295dcc6d0d 100644--- a/products/data_modeling/backend/test/test_preview_freshness_schedules.py+++ b/products/data_modeling/backend/test/test_preview_freshness_schedules.py@@ -9,9 +9,8 @@ from django.core.management import call_commandfrom products.data_modeling.backend.logic.node_frequency import set_declared_targetfrom products.data_modeling.backend.models.dag import DAG-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQueryfrom products.data_modeling.backend.models.edge import Edge-from products.data_modeling.backend.models.node import Node, NodeType+from products.data_modeling.backend.models.node import NodeTypefrom products.data_modeling.backend.test.helpers import (no_existing_schedules as _no_existing_schedules,saved_query_node as _saved_query_node,@@ -138,25 +137,3 @@ class TestPreviewFreshnessSchedules(BaseTest):self.assertNotIn("Effective cadences", summary.getvalue())self.assertIn("Effective cadences", verbose.getvalue())self.assertIn("0:15:00", verbose.getvalue())-- def test_cross_dag_duplication_flags_unsafe_drop(self):- # one saved query materialized by two DAGs is double-materialized; a query unique to a DAG- # makes that DAG unsafe to drop, one whose queries all live elsewhere is safe- shared = DataWarehouseSavedQuery.objects.create(- name="shared_mv", team=self.team, query={"query": "SELECT 1", "kind": "HogQLQuery"}- )- default_dag = DAG.objects.create(team=self.team, name="Default")- canonical_dag = DAG.objects.create(team=self.team, name="posthog_team")- Node.objects.create(team=self.team, dag=default_dag, saved_query=shared, type=NodeType.MAT_VIEW)- Node.objects.create(team=self.team, dag=canonical_dag, saved_query=shared, type=NodeType.MAT_VIEW)- _saved_query_node(self.team, default_dag, "only_in_default", NodeType.MAT_VIEW)-- module = "products.data_modeling.backend.logic.schedule_reconcile"- out = StringIO()- with mock.patch(f"{module}.async_connect", new=mock.AsyncMock(return_value=_no_existing_schedules())):- call_command("preview_freshness_schedules", "--team-id", str(self.team.pk), stdout=out)-- output = out.getvalue()- self.assertIn("in >1 DAG (double-materialized): 1", output)- self.assertIn("only in Default (unsafe to drop): 1", output)- self.assertIn("only in posthog_team (safe to drop): 0", output)diff --git a/products/data_modeling/backend/test/test_repoint_managed_view_readers.py b/products/data_modeling/backend/test/test_repoint_managed_view_readers.pydeleted file mode 100644index 05e387455e8e..000000000000--- a/products/data_modeling/backend/test/test_repoint_managed_view_readers.py+++ /dev/null@@ -1,88 +0,0 @@-from io import StringIO--from posthog.test.base import BaseTest-from unittest.mock import patch--from django.core.management import call_command--from parameterized import parameterized--from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag-from products.data_modeling.backend.models import DataModelingJob, DataModelingJobStatus, Edge, Node-from products.data_modeling.backend.models.dag import DAG-from products.data_modeling.backend.models.datawarehouse_managed_viewset import DataWarehouseManagedViewSet-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery-from products.data_modeling.backend.models.node import NodeType-from products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKind--MANAGED_VIEWSETS_FLAG = "managed-viewsets"---class TestRepointManagedViewReaders(BaseTest):- def setUp(self):- super().setUp()- flag_patch = patch(- "posthog.hogql.database.database.feature_enabled_or_false",- side_effect=lambda flag, *args, **kwargs: flag == MANAGED_VIEWSETS_FLAG,- )- flag_patch.start()- self.addCleanup(flag_patch.stop)-- def _managed_view(self, name: str, sql: str, viewset: DataWarehouseManagedViewSet) -> DataWarehouseSavedQuery:- view = DataWarehouseSavedQuery.objects.create(- name=name,- team=self.team,- query={"query": sql, "kind": "HogQLQuery"},- columns={"event": "String"},- managed_viewset=viewset,- )- sync_saved_query_to_dag(view, dag=DAG.get_or_create_revenue_analytics(self.team), allow_managed=True)- return view-- @parameterized.expand(- [- ("idle", False, False),- ("charges_materializing", True, False),- ("reader_fails_to_sync", False, True),- ]- )- def test_only_apply_moves_readers_to_the_reference_and_deletes_the_copies(- self, _name: str, charges_materializing: bool, reader_fails: bool- ):- viewset = DataWarehouseManagedViewSet.objects.create(- team=self.team, kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS- )- charges = self._managed_view("revenue_charges", "SELECT event FROM events", viewset)- mrr = self._managed_view("revenue_mrr", "SELECT event FROM revenue_charges", viewset)- managed_dag = DAG.get_or_create_revenue_analytics(self.team)- managed_node_ids = set(Node.objects.filter(dag=managed_dag).values_list("id", flat=True))-- default_dag = DAG.get_or_create_default(self.team)- charges_copy = Node.objects.create(team=self.team, dag=default_dag, saved_query=charges, type=NodeType.MAT_VIEW)- mrr_copy = Node.objects.create(team=self.team, dag=default_dag, saved_query=mrr, type=NodeType.MAT_VIEW)- Edge.objects.create(team=self.team, dag=default_dag, source=charges_copy, target=mrr_copy)- user_view = DataWarehouseSavedQuery.objects.create(- name="user_view",- team=self.team,- query={"query": "" if reader_fails else "SELECT * FROM revenue_charges", "kind": "HogQLQuery"},- )- user_node = Node.objects.create(team=self.team, dag=default_dag, saved_query=user_view, type=NodeType.VIEW)- Edge.objects.create(team=self.team, dag=default_dag, source=charges_copy, target=user_node)- DataModelingJob.objects.create(team=self.team, saved_query=None, status=DataModelingJobStatus.RUNNING)- if charges_materializing:- DataModelingJob.objects.create(team=self.team, saved_query=charges, status=DataModelingJobStatus.RUNNING)-- call_command("repoint_managed_view_readers", team_id=self.team.pk, stdout=StringIO())- self.assertEqual(Node.objects.filter(id__in=[charges_copy.id, mrr_copy.id]).count(), 2)-- call_command("repoint_managed_view_readers", team_id=self.team.pk, apply=True, stdout=StringIO())-- remaining = set(Node.objects.filter(id__in=[charges_copy.id, mrr_copy.id]).values_list("id", flat=True))- self.assertEqual(remaining, {charges_copy.id} if charges_materializing or reader_fails else set())- user_source = Edge.objects.get(target=user_node).source- if reader_fails:- self.assertEqual(user_source.id, charges_copy.id)- else:- self.assertEqual(user_source.type, NodeType.TABLE)- self.assertEqual(user_source.properties["origin"], "cross_dag_view")- self.assertEqual(set(Node.objects.filter(dag=managed_dag).values_list("id", flat=True)), managed_node_ids)