Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-modeling): delete the finished dag consolidation commands (#118706)

Thiago Salvatore98e9d0eb9612
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index f7c8100ee990..c2f446532ff1 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -1416,7 +1416,6 @@
3 products/batch_exports/backend/temporal/pipeline/internal_stage.py
3 products/conversations/backend/temporal/zendesk_import/workflows.py
3 products/data_modeling/backend/logic/gitsync/config_parser.py
-3 products/data_modeling/backend/management/commands/consolidate_dags.py
3 products/data_warehouse/backend/logic/external_data_source/webhooks.py
3 products/experiments/stats/shared/statistics.py
3 products/marketing_analytics/backend/services/data_source_health.py
diff --git a/products/data_modeling/backend/logic/node_frequency.py b/products/data_modeling/backend/logic/node_frequency.py
index 359378be030b..8bf6e21d7c03 100644
--- a/products/data_modeling/backend/logic/node_frequency.py
+++ b/products/data_modeling/backend/logic/node_frequency.py
@@ -9,7 +9,7 @@ typed column.
import uuid
import dataclasses
-from collections.abc import Collection, Iterable
+from collections.abc import Iterable
from datetime import timedelta
from django.db.models import Q, QuerySet
@@ -356,13 +356,11 @@ def schedulable_nodes(dag: DAG) -> QuerySet[Node]:
return Node.objects.filter(dag=dag, type__in=SAVED_QUERY_NODE_TYPES).exclude(saved_query__deleted=True)
-def persist_seed_targets(
- dag: DAG, default: timedelta | None = None, only_node_ids: Collection[str] | None = None
-) -> int:
+def persist_seed_targets(dag: DAG, default: timedelta | None = None) -> int:
"""Persist a seed target on every schedulable node lacking one; never overwrites. Each seed is
normalized (see `normalize_seed_target`) so the persisted target equals what the scheduler will
run. `default` is the operator escape hatch for scheduled DAGs with no interval metadata
- anywhere. `only_node_ids` limits the seeding to those nodes. Returns how many targets were written.
+ anywhere. Returns how many targets were written.
"""
graph = build_frequency_graph(dag)
floors = all_source_floors(graph.edges, graph.source_intervals)
@@ -370,8 +368,6 @@ def persist_seed_targets(
written = 0
for node in schedulable_nodes(dag):
node_id = str(node.id)
- if only_node_ids is not None and node_id not in only_node_ids:
- continue
if get_declared_target(node) is not None:
continue
raw = seeds.get(node_id, default)
diff --git a/products/data_modeling/backend/logic/saved_query_dag_sync.py b/products/data_modeling/backend/logic/saved_query_dag_sync.py
index 593bf689cc13..ce70d040a226 100644
--- a/products/data_modeling/backend/logic/saved_query_dag_sync.py
+++ b/products/data_modeling/backend/logic/saved_query_dag_sync.py
@@ -12,8 +12,6 @@ from posthog.hogql.database.models import SavedQuery as HogQLSavedQuery
from posthog.hogql.database.s3_table import DataWarehouseTable as HogQLDataWarehouseTable
from posthog.hogql.errors import QueryError
-from posthog.dataclasses import frozen
-
from products.data_modeling.backend.facade.contracts import Dependent
from products.data_modeling.backend.facade.system_tables import DATA_MODELING_ALLOWED_SYSTEM_TABLES
from products.data_modeling.backend.logic.node_suspension import clear_suspension_if_query_changed
@@ -810,80 +808,3 @@ def promote_dag_view_nodes_to_matview(dag: DAG) -> int:
until that sweep and only goes dark afterwards.
"""
return _promote_view_nodes(Node.objects.filter(dag=dag))
-
-
-def stale_managed_view_copies() -> QuerySet[Node]:
- """Nodes of managed views outside the managed DAG, for views the managed DAG already holds."""
- managed_saved_query_ids = Node.objects.filter(
- dag__name=REVENUE_ANALYTICS_DAG_NAME, saved_query__isnull=False
- ).values("saved_query_id")
- return Node.objects.filter(
- saved_query__managed_viewset__isnull=False, saved_query_id__in=managed_saved_query_ids
- ).exclude(dag__name=REVENUE_ANALYTICS_DAG_NAME)
-
-
-@frozen
-class _Reader:
- node_id: UUID
- saved_query_id: UUID
-
-
-def _user_readers_of(copies: QuerySet[Node]) -> list[_Reader]:
- rows = (
- Node.objects.filter(
- incoming_edges__source__in=copies,
- saved_query__isnull=False,
- saved_query__managed_viewset__isnull=True,
- saved_query__deleted=False,
- )
- .values_list("id", "saved_query_id")
- .distinct()
- )
- return [_Reader(node_id=node_id, saved_query_id=saved_query_id) for node_id, saved_query_id in rows]
-
-
-def _repoint_reader(team_id: int, reader: _Reader, database: Database) -> None:
- from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-
- try:
- with transaction.atomic():
- saved_query = DataWarehouseSavedQuery.objects.select_for_update(no_key=True).get(
- team_id=team_id, id=reader.saved_query_id
- )
- node = Node.objects.select_related("dag").filter(team_id=team_id, id=reader.node_id).first()
- if node is None:
- return
- sync_saved_query_to_dag(saved_query, dag=node.dag, reconcile=False, database=database)
- except Exception:
- logger.exception("Could not repoint a reader of a stale managed view copy", node_id=str(reader.node_id))
-
-
-def _delete_unread_copies(team_id: int, dag: DAG, copies: QuerySet[Node]) -> None:
- materializing = DataModelingJob.objects.filter(
- team_id=team_id, status=DataModelingJobStatus.RUNNING, saved_query_id__isnull=False
- ).values("saved_query_id")
- unread = copies.filter(dag=dag, outgoing_edges__isnull=True).exclude(saved_query_id__in=materializing)
- deleted_any = False
- with transaction.atomic():
- lock_dag(team_id, dag.id)
- lock_dag_placements(team_id, dag.id)
- while unread.exists():
- delete_nodes(team_id, unread)
- deleted_any = True
- if deleted_any:
- maybe_reconcile_dag(dag)
-
-
-def retire_stale_managed_view_copies(team_id: int, copy_ids: Iterable[UUID]) -> None:
- copies = stale_managed_view_copies().filter(team_id=team_id, id__in=list(copy_ids))
- readers = _user_readers_of(copies)
- if readers:
- database = Database.create_for(
- team_id=team_id,
- bypass_warehouse_access_control=True,
- allowed_system_tables=DATA_MODELING_ALLOWED_SYSTEM_TABLES,
- )
- for reader in readers:
- _repoint_reader(team_id, reader, database)
- for dag in DAG.objects.filter(team_id=team_id, id__in=copies.values("dag_id")).order_by("id"):
- _delete_unread_copies(team_id, dag, copies)
diff --git a/products/data_modeling/backend/management/commands/backfill_dag_schedule_type.py b/products/data_modeling/backend/management/commands/backfill_dag_schedule_type.py
deleted file mode 100644
index e1c26c7d9872..000000000000
--- a/products/data_modeling/backend/management/commands/backfill_dag_schedule_type.py
+++ /dev/null
@@ -1,79 +0,0 @@
-from django.core.management.base import BaseCommand
-
-import structlog
-from asgiref.sync import async_to_sync
-from temporalio.client import ScheduleListActionStartWorkflow, ScheduleUpdate, ScheduleUpdateInput
-from temporalio.common import SearchAttributePair
-
-from posthog.temporal.common.client import async_connect
-from posthog.temporal.common.search_attributes import POSTHOG_SCHEDULE_TYPE_KEY
-
-from products.data_modeling.backend.schedule import DATA_MODELING_EXECUTE_DAG_WORKFLOW
-
-logger = structlog.get_logger(__name__)
-
-
-class Command(BaseCommand):
- help = "Stamp PostHogScheduleType on existing v2 execute-dag schedules so they can be found by a server-side filter"
-
- def add_arguments(self, parser):
- parser.add_argument(
- "--dry-run",
- action="store_true",
- default=False,
- help="Only report how many schedules would be stamped, without updating any",
- )
-
- def handle(self, *args, **options):
- dry_run = options["dry_run"]
- found, already_stamped, updated, failed = _backfill_schedule_type(dry_run)
- verb = "would stamp" if dry_run else "stamped"
- stamped = found - already_stamped if dry_run else updated
- logger.info(
- "Backfill complete",
- execute_dag_schedules=found,
- already_stamped=already_stamped,
- updated=updated,
- failed=failed,
- dry_run=dry_run,
- )
- self.stdout.write(
- f"execute-dag schedules found: {found}; already stamped: {already_stamped}; "
- f"{verb}: {stamped}; failed: {failed}"
- )
-
-
-async def _updater(input: ScheduleUpdateInput) -> ScheduleUpdate:
- merged = input.description.typed_search_attributes.updated(
- SearchAttributePair(key=POSTHOG_SCHEDULE_TYPE_KEY, value=DATA_MODELING_EXECUTE_DAG_WORKFLOW)
- )
- return ScheduleUpdate(schedule=input.description.schedule, search_attributes=merged)
-
-
-@async_to_sync
-async def _backfill_schedule_type(dry_run: bool) -> tuple[int, int, int, int]:
- temporal = await async_connect()
- found = 0
- already_stamped = 0
- updated = 0
- failed = 0
- async for listing in await temporal.list_schedules():
- action = listing.schedule.action if listing.schedule else None
- if not (
- isinstance(action, ScheduleListActionStartWorkflow)
- and action.workflow == DATA_MODELING_EXECUTE_DAG_WORKFLOW
- ):
- continue
- found += 1
- if listing.typed_search_attributes.get(POSTHOG_SCHEDULE_TYPE_KEY) == DATA_MODELING_EXECUTE_DAG_WORKFLOW:
- already_stamped += 1
- continue
- if dry_run:
- continue
- try:
- await temporal.get_schedule_handle(listing.id).update(_updater)
- updated += 1
- except Exception:
- failed += 1
- logger.exception("Error stamping schedule", schedule_id=listing.id)
- return found, already_stamped, updated, failed
diff --git a/products/data_modeling/backend/management/commands/backfill_missing_model_paths.py b/products/data_modeling/backend/management/commands/backfill_missing_model_paths.py
deleted file mode 100644
index 1274b5684c7c..000000000000
--- a/products/data_modeling/backend/management/commands/backfill_missing_model_paths.py
+++ /dev/null
@@ -1,137 +0,0 @@
-"""This command will soon become obsolete as we migrate every materialization to v2 backend."""
-
-import time
-import logging
-
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-
-import structlog
-
-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-from products.data_modeling.backend.models.modeling import DataWarehouseModelPath
-
-logger = structlog.get_logger(__name__)
-
-
-class Command(BaseCommand):
- help = (
- "Creates missing DataWarehouseModelPath records for materialized saved queries that have schedules but no paths"
- )
-
- def add_arguments(self, parser):
- parser.add_argument(
- "--team-ids",
- default=None,
- type=str,
- help="Comma separated list of team IDs to filter by",
- )
- parser.add_argument(
- "--start-after-team-id",
- default=None,
- type=int,
- help="Resume after this team ID (exclusive), following the team_id ordering used by this command",
- )
- parser.add_argument(
- "--batch-size",
- default=50,
- type=int,
- help="Number of saved queries to process per batch (default: 50)",
- )
- parser.add_argument(
- "--batch-delay",
- default=1.0,
- type=float,
- help="Seconds to wait between batches (default: 1.0)",
- )
- parser.add_argument(
- "--dry-run",
- action="store_true",
- default=False,
- help="Only show what would be updated without making changes",
- )
-
- def handle(self, **options):
- logger.setLevel(logging.INFO)
-
- queryset = DataWarehouseSavedQuery.objects.filter(
- sync_frequency_interval__isnull=False,
- ).exclude(deleted=True)
-
- if options.get("team_ids") is not None:
- try:
- team_ids = [int(tid) for tid in options["team_ids"].split(",")]
- except ValueError:
- raise CommandError("team-ids must be a comma separated list of team IDs")
- queryset = queryset.filter(team_id__in=team_ids)
-
- if options.get("start_after_team_id") is not None:
- queryset = queryset.filter(team_id__gt=options["start_after_team_id"])
-
- saved_queries = list(queryset.order_by("team_id", "id"))
-
- if not saved_queries:
- logger.info("No materialized saved queries found matching filters")
- return
-
- logger.info(f"Checking {len(saved_queries)} materialized saved queries for missing model paths")
-
- sq_ids_with_paths = set(
- DataWarehouseModelPath.objects.filter(saved_query__in=saved_queries).values_list(
- "saved_query_id", flat=True
- )
- )
- missing = [sq for sq in saved_queries if sq.id not in sq_ids_with_paths]
-
- logger.info(f"Found {len(missing)} saved queries with no model paths (out of {len(saved_queries)} total)")
-
- if not missing:
- logger.info("All saved queries have model paths, nothing to do")
- return
-
- if options["dry_run"]:
- for sq in missing:
- logger.info("Would create model paths", saved_query_id=str(sq.id), team_id=sq.team_id, name=sq.name)
- return
-
- if not settings.TEST:
- confirm = input(f"\n\tWill create model paths for {len(missing)} saved queries. Proceed? (y/n) ")
- if confirm.strip().lower() != "y":
- logger.info("Aborting")
- return
-
- batch_size = options["batch_size"]
- batch_delay = options["batch_delay"]
- created = 0
- failed = 0
-
- for batch_start in range(0, len(missing), batch_size):
- batch = missing[batch_start : batch_start + batch_size]
- batch_num = batch_start // batch_size + 1
- total_batches = (len(missing) + batch_size - 1) // batch_size
-
- logger.info(f"Processing batch {batch_num}/{total_batches} ({len(batch)} queries)")
-
- for sq in batch:
- try:
- sq.setup_model_paths()
- created += 1
- logger.info(
- "Created model paths",
- saved_query_id=str(sq.id),
- team_id=sq.team_id,
- name=sq.name,
- )
- except Exception:
- failed += 1
- logger.exception(
- "Error creating model paths",
- saved_query_id=str(sq.id),
- team_id=sq.team_id,
- )
-
- if batch_start + batch_size < len(missing):
- logger.info(f"Sleeping {batch_delay}s between batches")
- time.sleep(batch_delay)
-
- logger.info(f"Done! Created: {created}, Failed: {failed}, Total checked: {len(saved_queries)}")
diff --git a/products/data_modeling/backend/management/commands/backfill_saved_query_nodes.py b/products/data_modeling/backend/management/commands/backfill_saved_query_nodes.py
deleted file mode 100644
index 9ac53885312b..000000000000
--- a/products/data_modeling/backend/management/commands/backfill_saved_query_nodes.py
+++ /dev/null
@@ -1,156 +0,0 @@
-from collections.abc import Iterable
-from datetime import timedelta
-from itertools import groupby
-
-from django.core.exceptions import ObjectDoesNotExist
-from django.core.management.base import BaseCommand, CommandParser
-from django.db import transaction
-from django.db.models import Exists, OuterRef
-
-from posthog.hogql.database.database import Database
-from posthog.hogql.errors import BaseHogQLError
-
-from posthog.dataclasses import frozen
-from posthog.models import Team
-
-from products.data_modeling.backend.logic.node_frequency import persist_seed_targets
-from products.data_modeling.backend.logic.saved_query_dag_sync import (
- adopt_saved_query_without_edges,
- lineage_database,
- sync_saved_query_to_dag,
-)
-from products.data_modeling.backend.logic.schedule_reconcile import reconcile_dag_schedules
-from products.data_modeling.backend.logic.tier_run_report import MATERIALIZING_TYPES
-from products.data_modeling.backend.models.dag import DAG
-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-from products.data_modeling.backend.models.modeling import UnknownParentError
-from products.data_modeling.backend.models.node import Node
-
-DEFAULT_MATERIALIZED_TARGET = timedelta(days=1)
-ADOPTABLE_SYNC_ERRORS = (BaseHogQLError, UnknownParentError, ObjectDoesNotExist)
-
-
-@frozen
-class TeamBackfill:
- synced: int
- adopted: int
- seeded: int
- failed: dict[str, str]
-
-
-class Command(BaseCommand):
- help = (
- "Give every live saved query that has no DAG node a node in its team's default DAG. "
- "A query whose lineage does not resolve gets a node with no edges and a sync-failed marker. "
- "A materialized query gets a freshness target, its own interval or else its DAG's or else 1 day, "
- "so a tier schedule picks it up. Any other sync error is reported and left for a re-run. "
- "Each team's DAG schedules are reconciled afterwards, and always for a team passed with --team-id. "
- "Managed-viewset queries are skipped. Dry run unless --apply is passed."
- )
-
- def add_arguments(self, parser: CommandParser) -> None:
- parser.add_argument("--team-id", type=int, action="append", dest="team_ids", help="Limit to this team.")
- parser.add_argument("--apply", action="store_true", help="Write the nodes. Without it, only list them.")
-
- def handle(self, *args: object, team_ids: list[int] | None, apply: bool, **options: object) -> None:
- nodeless = nodeless_saved_queries(team_ids)
- teams = sorted({saved_query.team_id for saved_query in nodeless})
- self.stdout.write(f"{len(nodeless)} nodeless saved queries across {len(teams)} teams")
- if not apply:
- for saved_query in nodeless:
- self.stdout.write(f" team {saved_query.team_id}: {saved_query.name} ({saved_query.id})")
- return
-
- dags_to_reconcile = {
- team.pk: DAG.get_or_create_default(team) for team in Team.objects.filter(id__in=team_ids or [])
- }
- for team_id, grouped in groupby(nodeless, key=lambda saved_query: saved_query.team_id):
- team_queries = list(grouped)
- dag = DAG.get_or_create_default(team_queries[0].team)
- self._report(team_id, backfill_team(team_queries, dag))
- dags_to_reconcile[team_id] = dag
- for team_id, dag in dags_to_reconcile.items():
- self._reconcile(team_id, dag)
-
- def _report(self, team_id: int, result: TeamBackfill) -> None:
- for name, error in result.failed.items():
- self.stderr.write(f" team {team_id}: FAILED {name}, re-run to retry: {error}")
- self.stdout.write(
- f" team {team_id}: synced {result.synced}, adopted without edges {result.adopted}, "
- f"failed {len(result.failed)}, seeded targets {result.seeded}"
- )
-
- def _reconcile(self, team_id: int, dag: DAG) -> None:
- try:
- applied = reconcile_dag_schedules(dag, require_tiered=True)
- except Exception as error:
- self.stderr.write(
- f" team {team_id}: schedule reconcile FAILED, re-run with --team-id {team_id} --apply: {error!r}"
- )
- return
- if not applied:
- self.stderr.write(f" team {team_id}: schedules not reconciled, the DAG still runs on a pre-tier schedule")
-
-
-def nodeless_saved_queries(team_ids: list[int] | None) -> list[DataWarehouseSavedQuery]:
- queryset = (
- DataWarehouseSavedQuery.objects.exclude(deleted=True)
- .filter(managed_viewset__isnull=True)
- .exclude(Exists(Node.objects.filter(saved_query_id=OuterRef("pk"))))
- .select_related("team")
- .order_by("team_id", "created_at")
- )
- if team_ids:
- queryset = queryset.filter(team_id__in=team_ids)
- return list(queryset)
-
-
-def backfill_team(saved_queries: list[DataWarehouseSavedQuery], dag: DAG) -> TeamBackfill:
- database = lineage_database(dag.team)
- with transaction.atomic():
- synced, unresolved = sync_until_stuck(saved_queries, dag, database)
- adoptable = {
- saved_query: error for saved_query, error in unresolved.items() if isinstance(error, ADOPTABLE_SYNC_ERRORS)
- }
- for saved_query, error in adoptable.items():
- adopt_saved_query_without_edges(saved_query.team_id, saved_query.id, str(error), dag_id=dag.id)
- # A child that only failed because its parent had no node can sync now that the parent has one.
- resynced, _ = sync_until_stuck(list(adoptable), dag, database)
- synced.extend(resynced)
- adoptable = {saved_query: error for saved_query, error in adoptable.items() if saved_query not in resynced}
- materializing_node_ids = Node.objects.filter(
- dag=dag, saved_query__in=[*synced, *adoptable], type__in=MATERIALIZING_TYPES
- ).values_list("id", flat=True)
- seeded = persist_seed_targets(
- dag,
- default=DEFAULT_MATERIALIZED_TARGET,
- only_node_ids={str(node_id) for node_id in materializing_node_ids},
- )
- return TeamBackfill(
- synced=len(synced),
- adopted=len(adoptable),
- seeded=seeded,
- failed={
- saved_query.name: repr(error) for saved_query, error in unresolved.items() if saved_query not in adoptable
- },
- )
-
-
-def sync_until_stuck(
- saved_queries: Iterable[DataWarehouseSavedQuery], dag: DAG, database: Database
-) -> tuple[list[DataWarehouseSavedQuery], dict[DataWarehouseSavedQuery, Exception]]:
- pending = list(saved_queries)
- synced: list[DataWarehouseSavedQuery] = []
- while True:
- failures: dict[DataWarehouseSavedQuery, Exception] = {}
- for saved_query in pending:
- try:
- with transaction.atomic():
- sync_saved_query_to_dag(saved_query, dag=dag, reconcile=False, database=database)
- except Exception as error:
- failures[saved_query] = error
- continue
- synced.append(saved_query)
- if not failures or len(failures) == len(pending):
- return synced, failures
- pending = list(failures)
diff --git a/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py b/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py
index 7ddef4da482e..28833a2d3a0c 100644
--- a/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py
+++ b/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py
@@ -56,8 +56,7 @@ def has_live_dependents(saved_query: DataWarehouseSavedQuery) -> bool:
`delete_node_from_dag` refuses the delete on the same condition. This is the cheap pre-filter
for it: a query with a live dependent is skipped before the delete takes an advisory lock on
- every DAG the query has a node in. Nodes in several DAGs is the population `consolidate_dags`
- exists for.
+ every DAG the query has a node in.
"""
nodes = Node.objects.filter(team_id=saved_query.team_id, saved_query=saved_query)
return (
diff --git a/products/data_modeling/backend/management/commands/consolidate_dags.py b/products/data_modeling/backend/management/commands/consolidate_dags.py
deleted file mode 100644
index 113523172999..000000000000
--- a/products/data_modeling/backend/management/commands/consolidate_dags.py
+++ /dev/null
@@ -1,849 +0,0 @@
-import uuid
-import dataclasses
-from collections import defaultdict, deque
-from datetime import timedelta
-from typing import Any
-
-from django.conf import settings
-from django.core.exceptions import ObjectDoesNotExist
-from django.core.management.base import BaseCommand, CommandError, CommandParser
-from django.db import transaction
-from django.db.models import QuerySet
-
-import structlog
-from temporalio.client import Client
-
-from posthog.hogql.errors import BaseHogQLError
-
-from posthog.models.team import Team
-from posthog.temporal.common.client import sync_connect
-
-from products.data_modeling.backend.logic.cohort_scheduling import is_tier_schedule_id
-from products.data_modeling.backend.logic.freshness import (
- STREAMING,
- all_source_floors,
- format_cadence,
- normalize_seed_target,
-)
-from products.data_modeling.backend.logic.node_frequency import (
- build_frequency_graph,
- get_declared_target,
- schedulable_nodes,
- set_declared_target,
-)
-from products.data_modeling.backend.logic.saved_query_dag_sync import (
- move_node_out_of_managed_dag,
- node_type_for,
- sync_saved_query_to_dag,
-)
-from products.data_modeling.backend.logic.saved_query_reads import PROXY_SAVED_QUERY_ID_PROPERTY
-from products.data_modeling.backend.logic.schedule_reconcile import (
- delete_dag_schedules,
- delete_v1_saved_query_schedules,
- list_existing_schedule_ids,
- null_saved_query_intervals,
- reconcile_dag_schedules,
-)
-from products.data_modeling.backend.models.dag import DAG, DEFAULT_DAG_NAME, REVENUE_ANALYTICS_DAG_NAME
-from products.data_modeling.backend.models.data_modeling_job import DataModelingJob, DataModelingJobStatus
-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-from products.data_modeling.backend.models.edge import Edge
-from products.data_modeling.backend.models.modeling import UnknownParentError
-from products.data_modeling.backend.models.node import Node, NodeType
-
-logger = structlog.get_logger(__name__)
-
-# How the target DAG is currently scheduled — decides which Temporal work is safe.
-MODE_TIERED = "tiered" # per-cadence tier schedules ({dag_id}:{seconds})
-MODE_LEGACY_V2 = "legacy-v2" # single whole-DAG execute-dag schedule (id == dag_id)
-MODE_V1 = "v1-only" # no execute-dag schedule; queries run on per-query v1 schedules
-
-# The sync failures --adopt-unresolvable may adopt: the query's SQL failing to parse or resolve
-# (BaseHogQLError covers parse errors and the BoundedResolver family), a parent that is not a
-# known table or view, or a parent whose node lives in a DAG the resolver cannot see (surfaces
-# as Node/SavedQuery DoesNotExist). Operational failures (DB errors, ManagedDAGError, a query
-# with no SQL at all) stay failed moves — a retry could still move those intact.
-ADOPTABLE_SYNC_ERRORS = (BaseHogQLError, UnknownParentError, ObjectDoesNotExist)
-
-
-class MoveItem:
- saved_query: DataWarehouseSavedQuery
- declared_target: timedelta | None
-
-
-class DropItem:
- saved_query_id: str
- node_id: str
- name: str
- declared_target: timedelta | None
-
-
-class SourceDagPlan:
- dag: DAG
- moves: list[MoveItem] # dependency (topological) order
- drops: list[DropItem]
- moved_node_ids: set[str] # kept for the dry run's edge-impact count
- dropped_node_ids: set[str]
-
-
-class Command(BaseCommand):
- help = (
- "Plan and (with --apply) execute collapsing a team's overlapping DAGs into one. The dry run "
- "is the planner: it partitions each source DAG's nodes into MOVE/DROP, counts the edges a "
- "merge would re-point or dedup, flags saved queries whose declared freshness targets differ "
- "between copies (a 'which cadence wins?' call for a human), and checks for anomalous "
- "cross-DAG edges. For every non-managed source DAG, saved queries already in the "
- "target are dropped (their duplicate node dies with the source DAG) and the rest are moved "
- "by re-syncing them into the target in dependency order, then the source DAG is deleted "
- "(cascading its nodes and edges). Tears down the source DAGs' execute-dag schedules, then "
- "finalizes the target from its persistent state: seed declared targets from leftover v1 "
- "intervals, sweep v1 per-query schedules, null the intervals, reconcile tiers once. "
- "Because the finalize derives from the target rather than this run's moves, re-running "
- "with --apply completes a partially-applied consolidation. A query whose SQL fails "
- "dependency resolution normally keeps its source DAG and fails the run; "
- "--adopt-unresolvable instead lands it in the target with no edges and a "
- "properties.system.degraded_sync marker. A system-managed DAG (Revenue Analytics) is "
- "folded in as a source too: its schedules are torn down before its nodes move, and the "
- "proxy table nodes that stood in for its views are replaced by the real nodes. "
- "--teams-with-managed-dag runs the fold for every team that still has a managed DAG and "
- "halts at the first team that fails. Dry run by default; pass --apply to execute."
- )
-
- def add_arguments(self, parser: CommandParser) -> None:
- teams = parser.add_mutually_exclusive_group(required=True)
- teams.add_argument("--team-id", type=int)
- teams.add_argument(
- "--teams-with-managed-dag",
- action="store_true",
- help="Run for every team that still has a system-managed DAG, halting at the first failure",
- )
- parser.add_argument(
- "--target-dag-id",
- type=str,
- default=None,
- help="Force the merge target DAG (default: the canonical 'Default' DAG, else the largest)",
- )
- parser.add_argument(
- "--apply",
- action="store_true",
- default=False,
- help="Execute the consolidation. Without this flag the command is a dry run and changes nothing.",
- )
- parser.add_argument("--yes", action="store_true", default=False, help="Skip the confirmation prompt")
- parser.add_argument(
- "--verbose",
- action="store_true",
- help="List per-query names under each source DAG's move/drop counts (default: counts only)",
- )
- parser.add_argument(
- "--adopt-unresolvable",
- action="store_true",
- default=False,
- help=(
- "When a query's SQL fails dependency resolution during a move, create its node in "
- "the target with no incoming edges (marked properties.system.degraded_sync) instead "
- "of keeping the source DAG and failing the run. The query keeps failing at "
- "materialization time as it already does; edges rebuild on its next successful "
- "sync. Caveat: until then the node runs at level 0, before any real parents."
- ),
- )
-
- def handle(self, *args: Any, **options: Any) -> None:
- if options["teams_with_managed_dag"]:
- self._fold_managed_dags_fleet(options)
- return
- self._consolidate_team(options["team_id"], options)
-
- def _fold_managed_dags_fleet(self, options: dict[str, Any]) -> None:
- team_ids = list(
- DAG.objects.filter(name=REVENUE_ANALYTICS_DAG_NAME)
- .order_by("team_id")
- .values_list("team_id", flat=True)
- .distinct()
- )
- self.stdout.write(f"{len(team_ids)} team(s) have a system-managed DAG")
- for processed, team_id in enumerate(team_ids):
- try:
- self._consolidate_team(team_id, options)
- except Exception as error:
- logger.exception("Failed to fold the managed DAG of a team", team_id=team_id)
- raise CommandError(
- f"halted at team {team_id} after {processed} team(s): {error}; "
- "fix it and re-run, finished teams no longer have a managed DAG"
- )
- self.stdout.write(f"\nProcessed {len(team_ids)} team(s)")
-
- def _consolidate_team(self, team_id: int, options: dict[str, Any]) -> None:
- apply_changes: bool = options["apply"]
- all_dags = list(DAG.objects.filter(team_id=team_id))
- if not all_dags:
- raise CommandError(f"Team {team_id} has no DAGs")
-
- managed_dags = [dag for dag in all_dags if dag.is_managed]
- dags = [dag for dag in all_dags if not dag.is_managed]
-
- forced_id: str | None = options["target_dag_id"]
- if forced_id:
- try:
- forced_uuid = uuid.UUID(forced_id)
- except ValueError:
- raise CommandError(f"--target-dag-id must be a UUID, got {forced_id!r}")
- if any(dag.id == forced_uuid for dag in managed_dags):
- raise CommandError(f"--target-dag-id {forced_id} is a system-managed DAG and cannot be a merge target")
-
- self.stdout.write(f"\nTeam {team_id} — {len(all_dags)} DAG(s)")
- if managed_dags:
- folded = ", ".join(f"{dag.name} ({dag.id})" for dag in managed_dags)
- self.stdout.write(f"Folding {len(managed_dags)} system-managed DAG(s) into the target: {folded}")
-
- if not dags and not forced_id:
- if not apply_changes:
- self.stdout.write(
- "No Default DAG yet — --apply creates it as the merge target (dry run: nothing changed)"
- )
- return
- target = DAG.get_or_create_default(Team.objects.get(id=team_id))
- else:
- target = self._pick_target(dags, forced_id)
- source_dags = sorted(
- (dag for dag in all_dags if dag.id != target.id), key=lambda dag: (not dag.is_managed, dag.name)
- )
- self.stdout.write(f"Merge target: {target.name} ({target.id}) — {self._target_reason(target, options)}")
-
- plans = self._build_plan(target, source_dags)
- self._print_plan(target, plans, team_id=team_id, dags=all_dags, verbose=options["verbose"])
-
- if not apply_changes:
- self.stdout.write(
- "\n(dry run — nothing was changed; target schedule mode is detected at apply time; "
- "pass --apply to execute)"
- )
- return
-
- if not settings.TEST and not options["yes"]:
- confirm = input(
- f"\n\tWill consolidate {len(source_dags)} source DAG(s) into {target.name} ({target.id}) "
- f"for team {team_id}, deleting the source DAGs. Proceed? (y/n) "
- )
- if confirm.strip().lower() != "y":
- self.stdout.write("Aborting")
- return
-
- # Connect before mutating anything: schedule teardown is mandatory once DAGs start moving,
- # so a Temporal outage must abort the run while it is still a pure no-op.
- try:
- temporal = sync_connect()
- except Exception as error:
- raise CommandError(f"Cannot connect to Temporal, aborting before any change: {error}")
- mode = self._detect_target_mode(self._list_dag_schedules_or_abort(str(target.id)))
- self.stdout.write(f"\nTarget schedule mode: {mode}")
- if mode == MODE_V1:
- # A v2-scheduled source would strand its queries: its execute-dag schedules die with
- # the source DAG, but a v1-only target has no schedule to hand coverage to.
- v2_sources = [
- f"{plan.dag.name} ({plan.dag.id})"
- for plan in plans
- if not plan.dag.is_managed and self._list_dag_schedules_or_abort(str(plan.dag.id))
- ]
- if v2_sources:
- raise CommandError(
- f"target {target.name} ({target.id}) is v1-only but source(s) "
- f"{', '.join(v2_sources)} are v2-scheduled — migrate the target to tiers first "
- "via reconcile_freshness_schedules, or choose a tiered target"
- )
- self.stdout.write(
- " target has no execute-dag schedule and every source is v1-only — graph-only "
- "consolidation: v1 per-query schedules and sync intervals are kept so the moved "
- "queries stay scheduled"
- )
-
- consolidated_sq_ids: set[str] = set()
- kept_dags: list[str] = []
- adopted_names: list[str] = []
- for plan in plans:
- kept_reason = self._apply_source_dag(
- plan,
- target,
- temporal,
- consolidated_sq_ids,
- adopted_names,
- adopt_unresolvable=options["adopt_unresolvable"],
- )
- if kept_reason:
- kept_dags.append(f"{plan.dag.name} ({plan.dag.id}): {kept_reason}")
-
- if not kept_dags and target.name != DEFAULT_DAG_NAME:
- self._rename_to_default(target)
- proxies_left = self._replace_proxy_nodes(target)
- try:
- v1_failed_sq_ids = self._finalize_target(target, temporal, bootstrap_tiers=bool(managed_dags))
- except Exception as error:
- if not managed_dags or kept_dags:
- raise
- # The managed DAG is what makes a run create the first tier schedules, and it is gone now.
- raise CommandError(
- f"{error}; re-running will not create the target's first tier schedules because the "
- f"managed DAG is gone, so run reconcile_freshness_schedules --team-id {team_id} to converge them"
- ) from error
-
- self.stdout.write(f"\nDone: consolidated {len(consolidated_sq_ids)} saved query(ies) into {target.name}")
- if adopted_names:
- self.stdout.write(
- self.style.WARNING(
- f" adopted {len(adopted_names)} query(ies) without edges (SQL failed to resolve): "
- f"{', '.join(sorted(adopted_names))} — they run at level 0 until their next "
- "successful sync rebuilds edges"
- )
- )
- if v1_failed_sq_ids:
- self.stdout.write(
- self.style.WARNING(
- f" v1 schedule delete failed for {len(v1_failed_sq_ids)} saved query(ies) "
- f"(intervals kept for retry): {', '.join(sorted(v1_failed_sq_ids))}"
- )
- )
- if kept_dags:
- for line in kept_dags:
- self.stdout.write(self.style.ERROR(f" kept source DAG {line}"))
- raise CommandError(
- f"consolidation incomplete: {len(kept_dags)} source DAG(s) kept; "
- "resolve the failures above and re-run (the command converges on re-run)"
- )
- if proxies_left:
- raise CommandError(
- f"consolidation incomplete: proxy node(s) {', '.join(sorted(proxies_left))} still stand in for "
- "views whose node never reached the target; re-run once those views have moved"
- )
-
- def _pick_target(self, dags: list[DAG], target_dag_id: str | None) -> DAG:
- if target_dag_id:
- try:
- wanted = uuid.UUID(target_dag_id)
- except ValueError:
- raise CommandError(f"--target-dag-id must be a UUID, got {target_dag_id!r}")
- for dag in dags:
- if dag.id == wanted:
- return dag
- raise CommandError(f"--target-dag-id {target_dag_id} is not a DAG of this team")
-
- for dag in dags:
- if dag.name == DEFAULT_DAG_NAME:
- return dag
-
- counts: dict[Any, int] = defaultdict(int)
- for dag_id in Node.objects.filter(dag__in=dags).values_list("dag_id", flat=True):
- counts[dag_id] += 1
- return sorted(dags, key=lambda d: (-counts.get(d.id, 0), d.name))[0]
-
- def _target_reason(self, target: DAG, options: dict[str, Any]) -> str:
- if options["target_dag_id"]:
- return "chosen via --target-dag-id"
- if target.name == DEFAULT_DAG_NAME:
- return "canonical 'Default' DAG"
- return "no Default DAG — largest DAG by node count"
-
- def _build_plan(self, target: DAG, source_dags: list[DAG]) -> list[SourceDagPlan]:
- """Partition every source DAG's schedulable nodes into MOVE/DROP and
- topologically order the moves, all read-only so any cycle aborts before a single mutation.
- """
- target_sq_ids = {str(sq_id) for sq_id in schedulable_nodes(target).values_list("saved_query_id", flat=True)}
- claimed: set[str] = set() # saved queries a preceding source DAG already moves
- placed_in_managed_dag = {
- str(sq_id)
- for sq_id in Node.objects.filter(
- team=target.team, dag__name=REVENUE_ANALYTICS_DAG_NAME, saved_query__isnull=False
- ).values_list("saved_query_id", flat=True)
- }
- plans: list[SourceDagPlan] = []
- for dag in source_dags:
- moves_by_node: dict[str, MoveItem] = {}
- drops: list[DropItem] = []
- floors = self._source_floors(dag) if dag.sync_frequency_interval is not None else {}
- for node in schedulable_nodes(dag).select_related("saved_query"):
- saved_query = node.saved_query
- if saved_query is None or (not dag.is_managed and str(saved_query.id) in placed_in_managed_dag):
- continue
- sq_id = str(saved_query.id)
- # Query-interval intent survives the move on the saved query itself, and a moved
- # no-interval query adopts the target DAG's default cadence — both seeded in the
- # finalize. What has no carrier is the case where the target has no default either:
- # cadence then lives only on the *source* DAG's interval, which the finalize can't
- # see, so capture it as a declared target before the source DAG — the only place it
- # exists — is deleted; without it the moved node has no effective cadence and
- # reconcile drops it from every tier.
- declared = get_declared_target(node)
- if (
- declared is None
- and saved_query.sync_frequency_interval is None
- and target.sync_frequency_interval is None
- and dag.sync_frequency_interval is not None
- ):
- declared = normalize_seed_target(dag.sync_frequency_interval, floors.get(str(node.id), STREAMING))
- if sq_id in target_sq_ids or sq_id in claimed:
- drops.append(
- DropItem(saved_query_id=sq_id, node_id=str(node.id), name=node.name, declared_target=declared)
- )
- else:
- moves_by_node[str(node.id)] = MoveItem(saved_query=saved_query, declared_target=declared)
- claimed.add(sq_id)
- order = self._dependency_order(dag, set(moves_by_node))
- plans.append(
- SourceDagPlan(
- dag=dag,
- moves=[moves_by_node[node_id] for node_id in order],
- drops=drops,
- moved_node_ids=set(moves_by_node),
- dropped_node_ids={drop.node_id for drop in drops},
- )
- )
- return plans
-
- def _source_floors(self, dag: DAG) -> dict[str, timedelta]:
- graph = build_frequency_graph(dag)
- return all_source_floors(graph.edges, graph.source_intervals)
-
- def _cadence_intent(self, node: Node, dag: DAG, floors: dict[str, timedelta]) -> timedelta | None:
- """A node's cadence intent, resolved in the same order as `seed_targets`.
-
- The DAG-interval fallback matters because the finalize nulls every query interval before
- reconciling: a query that was only ever scheduled by its DAG's default cadence would then
- have no declared target and no interval, and `reconcile_dag_schedules` drops a node with no
- effective cadence from every tier — it would stop materializing with nothing left to show why.
- """
- interval = None
- if node.saved_query is not None and node.saved_query.sync_frequency_interval is not None:
- interval = node.saved_query.sync_frequency_interval
- elif dag.sync_frequency_interval is not None:
- interval = dag.sync_frequency_interval
- if interval is None:
- return None
- return normalize_seed_target(interval, floors.get(str(node.id), STREAMING))
-
- def _dependency_order(self, dag: DAG, node_ids: set[str]) -> list[str]:
- """Kahn's algorithm over the source DAG's edges restricted to the moved nodes: a moved
- query's node must exist in the target before a moved dependent's sync resolves it. The
- model forbids cycles, but a corrupt graph must abort here, before anything is mutated.
- """
- children: dict[str, list[str]] = defaultdict(list)
- in_degree = dict.fromkeys(node_ids, 0)
- for source_id, target_id in Edge.objects.filter(dag=dag).values_list("source_id", "target_id"):
- src, tgt = str(source_id), str(target_id)
- if src in node_ids and tgt in node_ids:
- children[src].append(tgt)
- in_degree[tgt] += 1
- queue = deque(sorted(node_id for node_id, degree in in_degree.items() if degree == 0))
- order: list[str] = []
- while queue:
- node_id = queue.popleft()
- order.append(node_id)
- for child in children[node_id]:
- in_degree[child] -= 1
- if in_degree[child] == 0:
- queue.append(child)
- if len(order) != len(node_ids):
- raise CommandError(f"cycle detected among nodes to move in DAG {dag.id}; aborting before any change")
- return order
-
- def _print_plan(
- self, target: DAG, plans: list[SourceDagPlan], *, team_id: int, dags: list[DAG], verbose: bool
- ) -> None:
- if not plans:
- self.stdout.write(" Only one DAG — nothing to consolidate.")
- return
- total_moved = 0
- total_dropped = 0
- total_repoint_edges = 0
- total_dedup_edges = 0
- for plan in plans:
- total_moved += len(plan.moves)
- total_dropped += len(plan.drops)
- repoint, dedup = self._count_edge_impact(plan.dag, plan.moved_node_ids, plan.dropped_node_ids)
- total_repoint_edges += repoint
- total_dedup_edges += dedup
- self.stdout.write(f"\n Source DAG {plan.dag.name} ({plan.dag.id}):")
- self.stdout.write(f" move {len(plan.moves)}, drop {len(plan.drops)}")
- self.stdout.write(f" edges to re-point (touch a moved node): {repoint}")
- self.stdout.write(f" edges to drop as redundant (only dropped nodes): {dedup}")
- self.stdout.write(
- f" v1 schedules swept in the target-wide finalize (if target is v2-scheduled): "
- f"{len(plan.moves) + len(plan.drops)}"
- )
- self.stdout.write(f" execute-dag schedules to tear down: all listed for DAG {plan.dag.id}")
- if verbose:
- if plan.moves:
- move_names = ", ".join(item.saved_query.name for item in plan.moves)
- self.stdout.write(f" move (in dependency order): {move_names}")
- if plan.drops:
- self.stdout.write(f" drop: {', '.join(sorted(drop.name for drop in plan.drops))}")
-
- conflicts = self._find_target_conflicts(dags)
- self._print_conflicts(conflicts)
- cross_dag_edges = self._count_cross_dag_edges(team_id)
-
- target_after = schedulable_nodes(target).count() + total_moved
- self.stdout.write(f"\nSummary for target {target.name} ({target.id}):")
- self.stdout.write(f" nodes in target after merge: {target_after}")
- self.stdout.write(f" moved: {total_moved}, dropped: {total_dropped}")
- self.stdout.write(f" edges to re-point: {total_repoint_edges}, edges to drop: {total_dedup_edges}")
- self.stdout.write(f" source DAGs to delete: {len(plans)}")
- self.stdout.write(f" proxy nodes to replace: {self._proxy_nodes(target).count()}")
- self.stdout.write(f" freshness-target conflicts: {len(conflicts)}")
- if cross_dag_edges:
- self.stdout.write(self.style.WARNING(f" ⚠ anomalous cross-DAG edges found: {cross_dag_edges}"))
- else:
- self.stdout.write(" cross-DAG edges: 0 (edges are within-DAG by construction)")
- self.stdout.write(" target reconciled once at the end (if on cadence tiers)")
-
- def _count_edge_impact(self, dag: DAG, moved_ids: set[str], dropped_ids: set[str]) -> tuple[int, int]:
- """Count edges a real merge would touch. Edges are within-DAG (source/target/edge all share
- the DAG), so an edge touching a moved node must be re-pointed to the target DAG; an edge
- whose only involved nodes are dropped is redundant against the target's existing edge.
- """
- repoint = 0
- dedup = 0
- for source_id, target_id in Edge.objects.filter(dag=dag).values_list("source_id", "target_id"):
- src, tgt = str(source_id), str(target_id)
- if src in moved_ids or tgt in moved_ids:
- repoint += 1
- elif src in dropped_ids or tgt in dropped_ids:
- dedup += 1
- return repoint, dedup
-
- def _find_target_conflicts(self, dags: list[DAG]) -> list[tuple[str, list[tuple[str, timedelta | None]]]]:
- """Saved queries with schedulable nodes in 2+ DAGs whose declared freshness targets differ.
- A conflict is 2+ distinct non-None declared targets across the copies — after a merge only
- one node survives, so a human must pick the winning cadence. All-unset or agreeing targets
- are not conflicts (unset means 'no opinion', so a lone declared target wins cleanly).
- """
- by_saved_query: dict[object, list[tuple[str, str, timedelta | None]]] = defaultdict(list)
- dag_names = {dag.id: dag.name for dag in dags}
- for node in Node.objects.filter(dag__in=dags).exclude(type=NodeType.TABLE).exclude(saved_query__deleted=True):
- if node.saved_query_id is None:
- continue
- by_saved_query[node.saved_query_id].append((dag_names[node.dag_id], node.name, get_declared_target(node)))
-
- conflicts: list[tuple[str, list[tuple[str, timedelta | None]]]] = []
- for copies in by_saved_query.values():
- if len(copies) < 2:
- continue
- declared = {target for _dag_name, _query_name, target in copies if target is not None}
- if len(declared) < 2:
- continue
- query_name = copies[0][1]
- copies_report = sorted((dag_name, target) for dag_name, _query_name, target in copies)
- conflicts.append((query_name, copies_report))
- return sorted(conflicts, key=lambda c: c[0])
-
- def _print_conflicts(self, conflicts: list[tuple[str, list[tuple[str, timedelta | None]]]]) -> None:
- self.stdout.write("\nFreshness-target conflicts (which cadence wins after merge?):")
- if not conflicts:
- self.stdout.write(" none")
- return
- for query_name, copies in conflicts:
- self.stdout.write(self.style.WARNING(f" ⚠ {query_name}:"))
- for dag_name, target in copies:
- label = format_cadence(target) if target is not None else "unset"
- self.stdout.write(f" {dag_name}: {label}")
-
- def _count_cross_dag_edges(self, team_id: int) -> int:
- """Defensive check: edges whose endpoints' DAGs disagree with the edge's own DAG. The model
- forbids these, so a non-zero count means data corruption worth surfacing before any merge.
- """
- node_dags = dict(Node.objects.filter(team_id=team_id).values_list("id", "dag_id"))
- anomalies = 0
- for source_id, target_id, dag_id in Edge.objects.filter(team_id=team_id).values_list(
- "source_id", "target_id", "dag_id"
- ):
- if node_dags.get(source_id) != dag_id or node_dags.get(target_id) != dag_id:
- anomalies += 1
- return anomalies
-
- def _list_dag_schedules_or_abort(self, dag_id: str) -> set[str]:
- """Authoritative listing of a DAG's execute-dag schedule ids via the PostHogDagId search
- attribute — never guessed from an id formula, so off-scheme schedules can't escape. Only
- used before the first mutation, where aborting is still a pure no-op.
- """
- try:
- return list_existing_schedule_ids(dag_id)
- except Exception as error:
- raise CommandError(f"Cannot list Temporal schedules for DAG {dag_id}, aborting before any change: {error}")
-
- def _detect_target_mode(self, target_schedule_ids: set[str]) -> str:
- if any(is_tier_schedule_id(schedule_id) for schedule_id in target_schedule_ids):
- return MODE_TIERED
- if target_schedule_ids:
- return MODE_LEGACY_V2
- return MODE_V1
-
- def _apply_source_dag(
- self,
- plan: SourceDagPlan,
- target: DAG,
- temporal: Client,
- consolidated_sq_ids: set[str],
- adopted_names: list[str],
- *,
- adopt_unresolvable: bool = False,
- ) -> str | None:
- """Consolidate one source DAG. Returns None on success, or the reason the DAG was kept.
-
- Ordering, chosen so a crash at any point leaves a state a re-run converges from:
- 1. (DB) sync each MOVE query into the target in dependency order — a crash here leaves
- duplicates the re-run classifies as DROP.
- 2. (Temporal) delete the source DAG's execute-dag schedules, from an authoritative
- listing — idempotent, so a crash after this is retried harmlessly.
- 3. (DB) delete the source DAG row, cascading its nodes and edges. This must come LAST:
- the row is the only durable pointer to the DAG's Temporal schedules, so deleting it
- before the teardown succeeded would orphan schedules forever.
- The consolidated queries' v1 schedules are swept afterwards by the target-wide finalize,
- which re-runs derive from the target's persistent state.
- """
- self.stdout.write(f"\n Source DAG {plan.dag.name} ({plan.dag.id}):")
- teardown_before_moves = plan.dag.is_managed
- if teardown_before_moves and self._has_running_materialization(plan):
- return "a materialization of one of its views is running; re-run once it finishes"
- if teardown_before_moves and not self._teardown_execute_dag_schedules(str(plan.dag.id)):
- return "execute-dag schedule teardown failed before any node moved; re-run to retry it"
- failed_moves: list[str] = []
- moved_sq_ids: list[str] = []
- for item in plan.moves:
- try:
- # Atomic, so a query whose dependencies fail to resolve leaves its node, edges and
- # marker in the managed DAG and the re-run retries it instead of finding it moved.
- with transaction.atomic():
- if plan.dag.is_managed:
- move_node_out_of_managed_dag(item.saved_query, target)
- node = sync_saved_query_to_dag(item.saved_query, dag=target, reconcile=False)
- except Exception as error:
- adoptable = adopt_unresolvable and isinstance(error, ADOPTABLE_SYNC_ERRORS)
- node = (
- self._adopt_without_edges(item.saved_query, target, error, from_managed_dag=plan.dag.is_managed)
- if adoptable
- else None
- )
- if node is None:
- failed_moves.append(item.saved_query.name)
- logger.exception(
- "Failed to move saved query into target DAG",
- saved_query_id=str(item.saved_query.id),
- source_dag_id=str(plan.dag.id),
- target_dag_id=str(target.id),
- )
- self.stderr.write(f" FAILED to move {item.saved_query.name}: {error}")
- continue
- self._carry_declared_target(node, item.declared_target, item.saved_query.name)
- moved_sq_ids.append(str(item.saved_query.id))
- adopted_names.append(item.saved_query.name)
- self.stdout.write(self.style.WARNING(f" adopted {item.saved_query.name} without edges: {error}"))
- continue
- if node is not None:
- self._carry_declared_target(node, item.declared_target, item.saved_query.name)
- moved_sq_ids.append(str(item.saved_query.id))
- self.stdout.write(f" moved {item.saved_query.name}")
- orphaned_drops: list[str] = []
- for drop in plan.drops:
- node = Node.objects.filter(dag=target, saved_query_id=drop.saved_query_id).first()
- if node is None:
- # The plan expected an earlier source DAG to have moved this query in. It did not,
- # so the copy about to be cascaded away is the only place its declared target
- # exists. Keep this DAG instead of destroying intent a re-run could still carry.
- orphaned_drops.append(drop.name)
- self.stderr.write(f" NOT dropping {drop.name}: its query never reached the target")
- continue
- if plan.dag.is_managed and drop.declared_target is not None:
- set_declared_target(node, drop.declared_target)
- else:
- self._carry_declared_target(node, drop.declared_target, drop.name)
- self.stdout.write(f" dropped duplicate {drop.name}")
-
- if failed_moves:
- # The successful moves are now duplicates a re-run classifies as DROP; deleting the DAG
- # here would destroy the only node of every query that failed to move.
- return f"{len(failed_moves)} move(s) failed ({', '.join(failed_moves)})"
- if orphaned_drops:
- return f"{len(orphaned_drops)} duplicate(s) never reached the target ({', '.join(orphaned_drops)})"
-
- if not teardown_before_moves and not self._teardown_execute_dag_schedules(str(plan.dag.id)):
- return "execute-dag schedule teardown failed; DAG kept so a re-run can retry it"
-
- plan.dag.delete()
- self.stdout.write(f" deleted source DAG {plan.dag.id}")
- consolidated_sq_ids.update(moved_sq_ids)
- consolidated_sq_ids.update(drop.saved_query_id for drop in plan.drops)
- return None
-
- def _adopt_without_edges(
- self, saved_query: DataWarehouseSavedQuery, target: DAG, error: Exception, *, from_managed_dag: bool = False
- ) -> Node | None:
- """Last-resort landing for a query whose SQL will not resolve: create its node in the
- target with no incoming edges so the consolidation can proceed and the source DAG can be
- deleted. The query keeps failing at materialization time exactly as it did in the source
- DAG (feeding the suspension circuit breaker), the marker makes these nodes findable
- fleet-wide, and the next successful sync clears the marker and rebuilds real edges.
- Returns None if even the bare node cannot be created, falling back to keeping the DAG.
- """
- try:
- # Atomic so a failed marker save rolls the creation back too: a committed but
- # unmarked node would make the re-run classify this query as DROP instead of
- # retrying the move, stranding it edge-less and invisible to the marker sweep.
- with transaction.atomic():
- if from_managed_dag:
- # The failed move rolled back, so bring the managed node over to keep its state.
- move_node_out_of_managed_dag(saved_query, target)
- node, _ = Node.objects.get_or_create(
- team_id=saved_query.team_id,
- saved_query=saved_query,
- dag=target,
- defaults={"name": saved_query.name, "type": node_type_for(saved_query)},
- )
- node.properties = node.properties or {}
- node.mark_lineage_sync_failed(str(error))
- node.save(update_fields=["properties"])
- return node
- except Exception:
- logger.exception(
- "Failed to adopt unresolvable saved query into target DAG",
- saved_query_id=str(saved_query.id),
- target_dag_id=str(target.id),
- )
- return None
-
- def _carry_declared_target(self, node: Node, declared: timedelta | None, name: str) -> None:
- """Carry a source node's declared freshness target onto its target-DAG node. The target
- DAG's copy wins a disagreement (the dry run surfaces these conflicts for a human first).
- """
- if declared is None:
- return
- existing = get_declared_target(node)
- if existing is None:
- set_declared_target(node, declared)
- elif existing != declared:
- self.stdout.write(
- f" cadence conflict on {name}: keeping target's {format_cadence(existing)} "
- f"(source declared {format_cadence(declared)})"
- )
-
- def _teardown_execute_dag_schedules(self, dag_id: str) -> bool:
- teardown = delete_dag_schedules(dag_id)
- for schedule_id in teardown.deleted:
- self.stdout.write(f" deleted execute-dag schedule {schedule_id}")
- if not teardown.ok:
- self.stderr.write(f" FAILED to tear down execute-dag schedules for DAG {dag_id}")
- return teardown.ok
-
- def _finalize_target(self, target: DAG, temporal: Client, *, bootstrap_tiers: bool = False) -> set[str]:
- """Converge the target after the moves: seed declared targets from leftover v1 intervals,
- sweep the v1 per-query schedules, null the consumed intervals, and reconcile the tier
- schedules once. Everything here derives from the target's persistent state — never this
- run's move list — so re-running the command completes a consolidation a crash left
- half-finished; that re-run IS the recovery mechanism, at the cost of a no-op run
- re-sweeping and re-reconciling the target.
-
- The schedule mode is re-detected here rather than taken from the pre-flight check: a
- concurrent reconcile can convert the target to tiers while the moves run, and acting on
- the stale v1-only snapshot would skip the sweep and leave overlapping v1 + tier schedules.
-
- On a legacy-v2 target the whole-DAG schedule (node_ids=None) picks the moved nodes up
- automatically, so only tiered targets are reconciled; the seeded declared targets preserve
- cadence intent for the eventual tier conversion either way. On a v1-only target nothing
- runs — every source was verified v1-only before any mutation, so the kept v1 schedules and
- intervals keep the moved queries scheduled. Returns the saved-query ids whose v1 delete
- failed (their intervals are kept as the retry signal).
- """
- try:
- mode = self._detect_target_mode(list_existing_schedule_ids(str(target.id)))
- except Exception as error:
- raise CommandError(
- f"consolidation applied but re-detecting target {target.id} schedule mode failed ({error}); "
- "re-run with --apply to finish the finalize"
- )
- self.stdout.write(f"\n finalize: target schedule mode {mode}")
- creates_first_tiers = mode == MODE_V1 and bootstrap_tiers
- if mode not in (MODE_TIERED, MODE_LEGACY_V2) and not creates_first_tiers:
- return set()
- self._seed_targets_from_intervals(target)
- nodes = list(schedulable_nodes(target).select_related("saved_query"))
- v1_failed = delete_v1_saved_query_schedules(
- nodes, team_id=target.team_id, dag_id=str(target.id), temporal=temporal
- )
- swept_ids = {str(node.saved_query_id) for node in nodes if node.saved_query_id is not None} - v1_failed
- cleared = null_saved_query_intervals(target, only_saved_query_ids=swept_ids)
- self.stdout.write(
- f"\n swept v1 schedules for {len(nodes)} saved query(ies) ({len(v1_failed)} failed), "
- f"cleared sync_frequency_interval on {cleared}"
- )
- if mode == MODE_TIERED or creates_first_tiers:
- self._reconcile_target(target, require_tiered=not creates_first_tiers)
- return v1_failed
-
- def _has_running_materialization(self, plan: SourceDagPlan) -> bool:
- return DataModelingJob.objects.filter(
- team_id=plan.dag.team_id,
- status=DataModelingJobStatus.RUNNING,
- saved_query_id__in=[item.saved_query.id for item in plan.moves]
- + [drop.saved_query_id for drop in plan.drops],
- ).exists()
-
- def _reconcile_target(self, target: DAG, *, require_tiered: bool) -> None:
- try:
- reconcile_dag_schedules(target, require_tiered=require_tiered)
- self.stdout.write(f" reconciled target DAG {target.id} tier schedules")
- except Exception as error:
- logger.exception("Failed to reconcile target DAG after consolidation", dag_id=str(target.id))
- raise CommandError(
- f"consolidation applied but the final reconcile of target {target.id} failed ({error}); "
- "run reconcile_freshness_schedules for this team to converge the tier schedules"
- )
-
- def _rename_to_default(self, target: DAG) -> None:
- """`DAG.get_or_create_default` looks the DAG up by name, so a team's only DAG must carry it."""
- self.stdout.write(f"\n renamed target {target.name} ({target.id}) to {DEFAULT_DAG_NAME}")
- target.name = DEFAULT_DAG_NAME
- target.save(update_fields=["name"])
-
- def _proxy_nodes(self, dag: DAG) -> QuerySet[Node]:
- return Node.objects.filter(dag=dag, type=NodeType.TABLE, properties__has_key=PROXY_SAVED_QUERY_ID_PROPERTY)
-
- def _replace_proxy_nodes(self, target: DAG) -> list[str]:
- """Point each proxy's readers at the real node now in the target, then delete the proxy.
-
- Returns the names of the proxies kept because their view has no node in the target.
- """
- kept: list[str] = []
- for proxy in self._proxy_nodes(target):
- real = Node.objects.filter(
- dag=target, saved_query_id=proxy.properties[PROXY_SAVED_QUERY_ID_PROPERTY]
- ).first()
- if real is None:
- kept.append(proxy.name)
- continue
- with transaction.atomic():
- for edge in Edge.objects.filter(dag=target, source=proxy):
- Edge.objects.get_or_create(
- team_id=target.team_id,
- dag=target,
- source=real,
- target_id=edge.target_id,
- defaults={"properties": edge.properties},
- )
- proxy.delete()
- self.stdout.write(f" replaced proxy {proxy.name} with its view's node")
- return kept
-
- def _seed_targets_from_intervals(self, target: DAG) -> None:
- """A query about to lose its v1 sync_frequency_interval must not lose its cadence intent:
- seed each target-DAG node's declared target from its query's interval (normalized, never
- overwriting) before the interval is nulled — persist_seed_targets/convert_dag_to_tiers in
- spirit, scoped to the target DAG and idempotent so crash-recovery re-runs converge.
- """
- floors = self._source_floors(target)
- for node in schedulable_nodes(target).select_related("saved_query"):
- if get_declared_target(node) is not None:
- continue
- seed = self._cadence_intent(node, target, floors)
- if seed is None:
- continue
- set_declared_target(node, seed)
diff --git a/products/data_modeling/backend/management/commands/preview_freshness_schedules.py b/products/data_modeling/backend/management/commands/preview_freshness_schedules.py
index bd6e409f312f..7fbb1b80bd9c 100644
--- a/products/data_modeling/backend/management/commands/preview_freshness_schedules.py
+++ b/products/data_modeling/backend/management/commands/preview_freshness_schedules.py
@@ -1,12 +1,11 @@
import uuid
-from collections import defaultdict
from django.core.management.base import BaseCommand, CommandError
from products.data_modeling.backend.logic.cohort_scheduling import format_tier, tier_sort_key
from products.data_modeling.backend.logic.schedule_reconcile import DagSchedulePreview, preview_dag_schedules
from products.data_modeling.backend.models.dag import DAG
-from products.data_modeling.backend.models.node import Node, NodeType
+from products.data_modeling.backend.models.node import Node
class Command(BaseCommand):
@@ -14,8 +13,7 @@ class Command(BaseCommand):
"Dry-run: show the per-cadence-tier schedules a DAG would get from its nodes' freshness "
"targets, and the create/update/delete plan against its current schedules. Reads only — "
"never creates, updates, or deletes a schedule, and touches no live read/write path. "
- "Prints a compact summary by default (--verbose for per-node cadences), plus a team-level "
- "cross-DAG duplication report to judge whether a redundant DAG is safe to drop."
+ "Prints a compact summary by default (--verbose for per-node cadences)."
)
def add_arguments(self, parser):
@@ -44,7 +42,6 @@ class Command(BaseCommand):
raise CommandError("No matching DAGs")
for dag in dags:
self._preview(dag, seed=options["seed"], verbose=options["verbose"])
- self._print_cross_dag_duplication(options["team_id"])
self.stdout.write("\n(dry run — no schedules were created, updated, or deleted)")
def _preview(self, dag: DAG, *, seed: bool, verbose: bool) -> None:
@@ -121,38 +118,3 @@ class Command(BaseCommand):
self.stdout.write(f" UPDATE {schedule_id} ({format_tier(tier)}, {len(node_ids)} nodes)")
for schedule_id in sorted(plan.to_delete):
self.stdout.write(f" DELETE {schedule_id}")
-
- def _print_cross_dag_duplication(self, team_id: int) -> None:
- """Which saved queries a team materializes in more than one DAG, and which are unique to
- each DAG. A query in >1 DAG is materialized redundantly; a DAG whose queries all also live
- elsewhere is safe to drop, one with unique queries is not."""
- dags = list(DAG.objects.filter(team_id=team_id))
- if len(dags) < 2:
- return
- dag_names = {dag.id: dag.name for dag in dags}
- rows = (
- Node.objects.filter(dag__in=dags)
- .exclude(type=NodeType.TABLE)
- .exclude(saved_query__deleted=True)
- .values_list("dag_id", "saved_query_id")
- )
- dags_by_query: dict[object, set[object]] = defaultdict(set)
- for dag_id, saved_query_id in rows:
- if saved_query_id is not None:
- dags_by_query[saved_query_id].add(dag_id)
-
- if not dags_by_query:
- return
- double_materialized = sum(1 for owning in dags_by_query.values() if len(owning) > 1)
- unique_per_dag: dict[object, int] = defaultdict(int)
- for owning in dags_by_query.values():
- if len(owning) == 1:
- unique_per_dag[next(iter(owning))] += 1
-
- self.stdout.write(f"\nTeam {team_id} cross-DAG duplication:")
- self.stdout.write(f" {len(dags_by_query)} distinct saved queries across {len(dags)} DAGs")
- self.stdout.write(f" in >1 DAG (double-materialized): {double_materialized}")
- for dag in sorted(dags, key=lambda d: dag_names[d.id]):
- unique = unique_per_dag.get(dag.id, 0)
- note = "safe to drop" if unique == 0 else "unsafe to drop"
- self.stdout.write(f" only in {dag_names[dag.id]} ({note}): {unique}")
diff --git a/products/data_modeling/backend/management/commands/repoint_managed_view_readers.py b/products/data_modeling/backend/management/commands/repoint_managed_view_readers.py
deleted file mode 100644
index eaac01f530be..000000000000
--- a/products/data_modeling/backend/management/commands/repoint_managed_view_readers.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from collections import defaultdict
-from typing import Any
-from uuid import UUID
-
-from django.core.management.base import BaseCommand, CommandParser
-from django.db.models import QuerySet
-
-import structlog
-
-from products.data_modeling.backend.logic.saved_query_dag_sync import (
- retire_stale_managed_view_copies,
- stale_managed_view_copies,
-)
-from products.data_modeling.backend.models.node import Node
-
-logger = structlog.get_logger(__name__)
-
-
-class Command(BaseCommand):
- help = (
- "Move readers of stale managed-view copies onto the cross-DAG reference, then delete the copies. "
- "Dry run unless --apply."
- )
-
- def add_arguments(self, parser: CommandParser) -> None:
- parser.add_argument("--team-id", type=int, default=None, help="Only clean up this team")
- parser.add_argument("--apply", action="store_true", default=False, help="Repoint and delete instead of listing")
-
- def handle(self, *args: Any, **options: Any) -> None:
- copy_ids_by_team = self._find_copies(options["team_id"])
- copy_count = sum(len(copy_ids) for copy_ids in copy_ids_by_team.values())
- self.stdout.write(f"{copy_count} stale managed view copies in {len(copy_ids_by_team)} team(s)")
- if not options["apply"]:
- return
-
- failed_team_ids: list[int] = []
- for team_id, copy_ids in copy_ids_by_team.items():
- try:
- retire_stale_managed_view_copies(team_id, copy_ids)
- except Exception:
- logger.exception("Could not retire stale managed view copies", team_id=team_id)
- failed_team_ids.append(team_id)
- self.stdout.write(f"Done. {self._copies(options['team_id']).count()} copies remain")
- if failed_team_ids:
- self.stderr.write(f"Failed for team(s): {', '.join(str(team_id) for team_id in failed_team_ids)}")
-
- def _copies(self, team_id: int | None) -> QuerySet[Node]:
- copies = stale_managed_view_copies()
- if team_id is not None:
- copies = copies.filter(team_id=team_id)
- return copies
-
- def _find_copies(self, team_id: int | None) -> dict[int, list[UUID]]:
- copies = self._copies(team_id)
- copy_ids_by_team: dict[int, list[UUID]] = defaultdict(list)
- for node_id, node_team_id, dag_id, name in copies.values_list("id", "team_id", "dag_id", "name").iterator():
- self.stdout.write(f"team={node_team_id} dag={dag_id} node={node_id} name={name}")
- copy_ids_by_team[node_team_id].append(node_id)
- return copy_ids_by_team
diff --git a/products/data_modeling/backend/models/node.py b/products/data_modeling/backend/models/node.py
index 2983852af52d..5bf5d2c84c86 100644
--- a/products/data_modeling/backend/models/node.py
+++ b/products/data_modeling/backend/models/node.py
@@ -25,7 +25,7 @@ SAVED_QUERY_NODE_TYPES = frozenset({NodeType.VIEW, NodeType.MAT_VIEW, NodeType.E
_SAVED_QUERY_NODE_TYPE_VALUES = sorted(node_type.value for node_type in SAVED_QUERY_NODE_TYPES)
-# properties["system"] marker set by consolidate_dags --adopt-unresolvable when a query's SQL
+# properties["system"] marker set by adopt_saved_query_without_edges when a query's SQL
# would not resolve and its node was created without edges. A successful sync clears it.
DEGRADED_SYNC_KEY = "degraded_sync"
diff --git a/products/data_modeling/backend/test/test_backfill_dag_schedule_type.py b/products/data_modeling/backend/test/test_backfill_dag_schedule_type.py
deleted file mode 100644
index 2fb1702c751a..000000000000
--- a/products/data_modeling/backend/test/test_backfill_dag_schedule_type.py
+++ /dev/null
@@ -1,55 +0,0 @@
-from unittest import mock
-
-from parameterized import parameterized
-from temporalio.client import ScheduleListActionStartWorkflow
-from temporalio.common import SearchAttributePair, TypedSearchAttributes
-
-from posthog.temporal.common.search_attributes import POSTHOG_SCHEDULE_TYPE_KEY
-
-from products.data_modeling.backend.management.commands.backfill_dag_schedule_type import _backfill_schedule_type
-from products.data_modeling.backend.schedule import DATA_MODELING_EXECUTE_DAG_WORKFLOW
-
-
-class TestBackfillScheduleType:
- def _listing(self, schedule_id: str, workflow: str, schedule_type: str | None = None):
- action = mock.Mock(spec=ScheduleListActionStartWorkflow, workflow=workflow)
- attrs = TypedSearchAttributes(
- [SearchAttributePair(POSTHOG_SCHEDULE_TYPE_KEY, schedule_type)] if schedule_type else []
- )
- return mock.Mock(id=schedule_id, schedule=mock.Mock(action=action), typed_search_attributes=attrs)
-
- def _temporal(self, listings):
- async def fake_list_schedules(*args, **kwargs):
- async def gen():
- for listing in listings:
- yield listing
-
- return gen()
-
- temporal = mock.Mock()
- temporal.list_schedules = fake_list_schedules
- temporal.get_schedule_handle = mock.Mock(return_value=mock.Mock(update=mock.AsyncMock()))
- return temporal
-
- @parameterized.expand(
- [
- ("dry_run", True, 0, []),
- ("real_run", False, 2, ["missing-1", "missing-2"]),
- ]
- )
- def test_counts_stamped_vs_missing_and_updates_only_missing(self, _name, dry_run, expected_updated, updated_ids):
- listings = [
- self._listing("already-stamped", DATA_MODELING_EXECUTE_DAG_WORKFLOW, DATA_MODELING_EXECUTE_DAG_WORKFLOW),
- self._listing("missing-1", DATA_MODELING_EXECUTE_DAG_WORKFLOW),
- self._listing("missing-2", DATA_MODELING_EXECUTE_DAG_WORKFLOW),
- self._listing("v1-schedule", "data-modeling-run"),
- ]
- temporal = self._temporal(listings)
- with mock.patch(
- "products.data_modeling.backend.management.commands.backfill_dag_schedule_type.async_connect",
- new=mock.AsyncMock(return_value=temporal),
- ):
- found, already_stamped, updated, failed = _backfill_schedule_type(dry_run)
-
- assert (found, already_stamped, updated, failed) == (3, 1, expected_updated, 0)
- assert [call.args[0] for call in temporal.get_schedule_handle.call_args_list] == updated_ids
diff --git a/products/data_modeling/backend/test/test_backfill_saved_query_nodes.py b/products/data_modeling/backend/test/test_backfill_saved_query_nodes.py
deleted file mode 100644
index 8cae8d080419..000000000000
--- a/products/data_modeling/backend/test/test_backfill_saved_query_nodes.py
+++ /dev/null
@@ -1,54 +0,0 @@
-from datetime import timedelta
-
-from posthog.test.base import BaseTest
-from unittest.mock import patch
-
-from django.core.management import call_command
-
-from products.data_modeling.backend.logic.node_frequency import get_declared_target
-from products.data_modeling.backend.models.dag import DAG
-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-from products.data_modeling.backend.models.edge import Edge
-from products.data_modeling.backend.models.node import LineageIssueKind, Node
-
-COMMAND = "products.data_modeling.backend.management.commands.backfill_saved_query_nodes"
-
-
-class TestBackfillSavedQueryNodes(BaseTest):
- def _view(
- self, name: str, sql: str, is_materialized: bool = False, interval: timedelta | None = None
- ) -> DataWarehouseSavedQuery:
- return DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name=name,
- query={"kind": "HogQLQuery", "query": sql},
- is_materialized=is_materialized,
- sync_frequency_interval=interval,
- )
-
- def test_gives_every_nodeless_view_a_node_only_when_applied(self) -> None:
- downstream = self._view("downstream_view", "SELECT event FROM upstream_view")
- upstream = self._view("upstream_view", "SELECT event FROM events")
- broken = self._view("broken_view", "SELECT 1 FROM no_such_table")
- materialized = self._view("materialized_view", "SELECT event FROM events", is_materialized=True)
- legacy_cadence = self._view(
- "legacy_cadence_view", "SELECT event FROM events", is_materialized=True, interval=timedelta(minutes=5)
- )
-
- with patch(f"{COMMAND}.reconcile_dag_schedules") as reconcile:
- call_command("backfill_saved_query_nodes")
- self.assertFalse(Node.objects.filter(team=self.team).exists())
-
- call_command("backfill_saved_query_nodes", "--apply")
-
- reconcile.assert_called_once_with(DAG.get_or_create_default(self.team), require_tiered=True)
- upstream_node = Node.objects.get(saved_query=upstream)
- downstream_node = Node.objects.get(saved_query=downstream)
- self.assertTrue(Edge.objects.filter(source=upstream_node, target=downstream_node).exists())
- broken_node = Node.objects.get(saved_query=broken)
- self.assertFalse(Edge.objects.filter(target=broken_node).exists())
- assert broken_node.lineage_issue is not None
- self.assertEqual(broken_node.lineage_issue["kind"], LineageIssueKind.SYNC_FAILED)
- self.assertEqual(get_declared_target(Node.objects.get(saved_query=materialized)), timedelta(days=1))
- self.assertEqual(get_declared_target(Node.objects.get(saved_query=legacy_cadence)), timedelta(minutes=15))
- self.assertIsNone(get_declared_target(upstream_node))
diff --git a/products/data_modeling/backend/test/test_consolidate_dags.py b/products/data_modeling/backend/test/test_consolidate_dags.py
deleted file mode 100644
index adcda6127698..000000000000
--- a/products/data_modeling/backend/test/test_consolidate_dags.py
+++ /dev/null
@@ -1,777 +0,0 @@
-import uuid
-from contextlib import contextmanager
-from datetime import timedelta
-from io import StringIO
-from types import SimpleNamespace
-
-import pytest
-from posthog.test.base import BaseTest
-from unittest import mock
-
-from django.core.management import call_command
-from django.core.management.base import CommandError
-
-from parameterized import parameterized
-from temporalio.client import ScheduleListActionStartWorkflow
-
-from posthog.models.team import Team
-
-from products.data_modeling.backend.logic.cohort_scheduling import tier_schedule_id
-from products.data_modeling.backend.logic.node_frequency import get_declared_target, set_declared_target
-from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag
-from products.data_modeling.backend.logic.schedule_reconcile import DagScheduleTeardown
-from products.data_modeling.backend.models.dag import DAG, DEFAULT_DAG_NAME, REVENUE_ANALYTICS_DAG_NAME
-from products.data_modeling.backend.models.data_modeling_job import DataModelingJob
-from products.data_modeling.backend.models.datawarehouse_managed_viewset import DataWarehouseManagedViewSet
-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-from products.data_modeling.backend.models.edge import Edge
-from products.data_modeling.backend.models.node import Node, NodeType
-from products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKind
-
-M15 = timedelta(minutes=15)
-H6 = timedelta(hours=6)
-H12 = timedelta(hours=12)
-
-COMMAND = "products.data_modeling.backend.management.commands.consolidate_dags"
-RECONCILE = "products.data_modeling.backend.logic.schedule_reconcile"
-
-
-def _listing_client(schedules_by_dag):
- # query-aware fake: schedule listings are per-DAG (PostHogDagId search attribute), and the
- # command lists the target, each source, and the reconcile path — they must not see each
- # other's schedules
- def _listing(schedule_id):
- action = mock.Mock(spec=ScheduleListActionStartWorkflow, workflow="data-modeling-execute-dag")
- return mock.Mock(id=schedule_id, schedule=mock.Mock(action=action))
-
- async def fake_list_schedules(*_args, query="", **_kwargs):
- dag_id = query.split("'")[1] if "'" in query else ""
-
- async def gen():
- for schedule_id in schedules_by_dag.get(dag_id, []):
- yield _listing(schedule_id)
-
- return gen()
-
- client = mock.Mock()
- client.list_schedules = fake_list_schedules
- return client
-
-
-@contextmanager
-def _temporal_boundary(schedules_by_dag=None, v2_teardown_fails=False):
- v2_delete = mock.Mock(return_value=DagScheduleTeardown(ok=not v2_teardown_fails, deleted=()))
- with (
- mock.patch(f"{COMMAND}.sync_connect"),
- mock.patch(f"{COMMAND}.delete_dag_schedules", v2_delete),
- mock.patch(f"{RECONCILE}.delete_schedule") as v1_delete,
- mock.patch(
- f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=_listing_client(schedules_by_dag or {}))
- ),
- mock.patch(f"{RECONCILE}.a_create_schedule", new=mock.AsyncMock()) as tier_create,
- mock.patch(f"{RECONCILE}.a_update_schedule", new=mock.AsyncMock()) as tier_update,
- mock.patch(f"{RECONCILE}.a_delete_schedule", new=mock.AsyncMock()),
- ):
- yield SimpleNamespace(
- v1_delete=v1_delete, v2_delete=v2_delete, tier_create=tier_create, tier_update=tier_update
- )
-
-
-class TestConsolidateDags(BaseTest):
- def _query(self, name: str, sql: str, interval: timedelta | None = None) -> DataWarehouseSavedQuery:
- return DataWarehouseSavedQuery.objects.create(
- name=name, team=self.team, query={"query": sql, "kind": "HogQLQuery"}, sync_frequency_interval=interval
- )
-
- def _node(self, dag: DAG, saved_query: DataWarehouseSavedQuery) -> Node:
- return Node.objects.create(team=self.team, dag=dag, saved_query=saved_query, type=NodeType.MAT_VIEW)
-
- def _run(self, *args: str, apply: bool = False) -> str:
- out = StringIO()
- err = StringIO()
- flags = ["--apply"] if apply else []
- call_command("consolidate_dags", "--team-id", str(self.team.pk), *flags, *args, stdout=out, stderr=err)
- return out.getvalue() + err.getvalue()
-
- def test_end_to_end_shared_dropped_unique_moved_with_rebuilt_edges(self):
- # The moved query depends on the dropped one, so its edges must rebuild against the
- # target's existing copy; the source DAG and both its nodes must be gone afterwards.
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- shared = self._query("shared_mv", "SELECT 1")
- target_shared_node = self._node(target, shared)
- self._node(source, shared)
- moved = self._query("only_in_source", "SELECT * FROM shared_mv")
- self._node(source, moved)
-
- with _temporal_boundary():
- output = self._run(apply=True)
-
- self.assertFalse(DAG.objects.filter(id=source.id).exists())
- moved_node = Node.objects.get(dag=target, saved_query=moved)
- self.assertTrue(Edge.objects.filter(dag=target, source=target_shared_node, target=moved_node).exists())
- self.assertEqual(Node.objects.filter(saved_query=shared).count(), 1)
- self.assertEqual(Node.objects.filter(saved_query=moved).count(), 1)
- self.assertIn("moved only_in_source", output)
- self.assertIn("dropped duplicate shared_mv", output)
- self.assertIn("deleted source DAG", output)
-
- def test_moves_sync_in_dependency_order(self):
- # dep_child depends on dep_parent, both only in the source: syncing the child first would
- # fail dependency resolution in the target (Node.DoesNotExist) and keep the source DAG.
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- parent = self._query("dep_parent", "SELECT * FROM events")
- child = self._query("dep_child", "SELECT * FROM dep_parent")
- parent_node = self._node(source, parent)
- child_node = self._node(source, child)
- Edge.objects.create(team=self.team, dag=source, source=parent_node, target=child_node)
-
- with _temporal_boundary():
- self._run(apply=True)
-
- self.assertFalse(DAG.objects.filter(id=source.id).exists())
- new_parent = Node.objects.get(dag=target, saved_query=parent)
- new_child = Node.objects.get(dag=target, saved_query=child)
- self.assertTrue(Edge.objects.filter(dag=target, source=new_parent, target=new_child).exists())
- events = Node.objects.get(dag=target, name="events", type=NodeType.TABLE)
- self.assertTrue(Edge.objects.filter(dag=target, source=events, target=new_parent).exists())
-
- @parameterized.expand(
- [
- ("tiered", "tier", True, True),
- ("legacy_v2", "legacy", True, False),
- ("v1_only", "none", False, False),
- ]
- )
- def test_target_schedule_mode_controls_teardown(self, _name, existing_kind, expect_sweep, expect_reconcile):
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- shared = self._query("shared_mv", "SELECT 1", interval=H6)
- self._node(target, shared)
- self._node(source, shared)
- moved = self._query("only_in_source", "SELECT * FROM events", interval=H6)
- self._node(source, moved)
-
- tier_id = tier_schedule_id(str(target.id), H6)
- schedules_by_dag = {
- "tier": {str(target.id): [tier_id]},
- "legacy": {str(target.id): [str(target.id)]},
- "none": {},
- }[existing_kind]
-
- with _temporal_boundary(schedules_by_dag=schedules_by_dag) as mocks:
- self._run(apply=True)
-
- self.assertFalse(DAG.objects.filter(id=source.id).exists())
- moved.refresh_from_db()
- shared.refresh_from_db()
- moved_node = Node.objects.get(dag=target, saved_query=moved)
- if expect_sweep:
- swept = {call.kwargs["schedule_id"] for call in mocks.v1_delete.call_args_list}
- self.assertEqual(swept, {str(moved.id), str(shared.id)})
- self.assertIsNone(moved.sync_frequency_interval)
- self.assertIsNone(shared.sync_frequency_interval)
- # cadence intent survives the nulled interval as a declared node target
- self.assertEqual(get_declared_target(moved_node), H6)
- else:
- mocks.v1_delete.assert_not_called()
- self.assertEqual(moved.sync_frequency_interval, H6)
- self.assertEqual(shared.sync_frequency_interval, H6)
- self.assertIsNone(get_declared_target(moved_node))
- if expect_reconcile:
- mocks.tier_update.assert_called_once()
- self.assertEqual(mocks.tier_update.call_args.kwargs["id"], tier_id)
- mocks.tier_create.assert_not_called()
- else:
- mocks.tier_update.assert_not_called()
- mocks.tier_create.assert_not_called()
-
- def test_refuses_v2_scheduled_source_when_target_is_v1_only(self):
- # moving these queries would strand them: their execute-dag schedules die with the source
- # DAG but a v1-only target has no schedule to hand coverage to
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- moved = self._query("only_in_source", "SELECT 1", interval=H6)
- self._node(source, moved)
-
- with _temporal_boundary(schedules_by_dag={str(source.id): [str(source.id)]}) as mocks:
- with self.assertRaisesRegex(CommandError, "v2-scheduled"):
- self._run(apply=True)
-
- self.assertTrue(DAG.objects.filter(id=source.id).exists())
- self.assertFalse(Node.objects.filter(dag=target, saved_query=moved).exists())
- moved.refresh_from_db()
- self.assertEqual(moved.sync_frequency_interval, H6)
- mocks.v1_delete.assert_not_called()
- mocks.v2_delete.assert_not_called()
-
- def test_rerun_finalizes_target_after_crashed_run(self):
- # a prior run that crashed after deleting its source DAGs leaves a moved query on the
- # target with its interval intact, no declared target, and a live v1 schedule; a plain
- # re-run must finish seed/sweep/null/reconcile from the target's own state
- target = DAG.get_or_create_default(self.team)
- stranded = self._query("stranded_mv", "SELECT 1", interval=H6)
- node = self._node(target, stranded)
- tier_id = tier_schedule_id(str(target.id), H6)
-
- with _temporal_boundary(schedules_by_dag={str(target.id): [tier_id]}) as mocks:
- self._run(apply=True)
-
- node.refresh_from_db()
- stranded.refresh_from_db()
- self.assertEqual(get_declared_target(node), H6)
- self.assertIsNone(stranded.sync_frequency_interval)
- swept = {call.kwargs["schedule_id"] for call in mocks.v1_delete.call_args_list}
- self.assertEqual(swept, {str(stranded.id)})
- mocks.tier_update.assert_called_once()
- self.assertEqual(mocks.tier_update.call_args.kwargs["id"], tier_id)
-
- def test_source_dag_interval_carries_as_declared_target(self):
- # Legacy-migrated source shape: query intervals nulled, cadence lives only on
- # dag.sync_frequency_interval. The finalize seeds from the *target* DAG, whose interval is
- # NULL here, so unless the source cadence is captured at plan time the moved query arrives
- # with no effective cadence and reconcile drops it from every tier.
- target = DAG.get_or_create_default(self.team)
- target.sync_frequency_interval = None
- target.save()
- source = DAG.objects.create(team=self.team, name="posthog_team", sync_frequency_interval=H6)
- moved = self._query("only_in_source", "SELECT 1")
- self._node(source, moved)
- tier_id = tier_schedule_id(str(target.id), H6)
-
- with _temporal_boundary(schedules_by_dag={str(target.id): [tier_id], str(source.id): [str(source.id)]}):
- self._run(apply=True)
-
- self.assertFalse(DAG.objects.filter(id=source.id).exists())
- moved_node = Node.objects.get(dag=target, saved_query=moved)
- self.assertEqual(get_declared_target(moved_node), H6)
-
- def test_finalize_redetects_target_mode_after_mid_run_conversion(self):
- # The pre-flight check sees a v1-only target; a concurrent reconcile converts it to tiers
- # before the finalize. Acting on the stale snapshot would skip the seed/sweep/reconcile and
- # leave the moved query double-scheduled (v1 + tier).
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- moved = self._query("only_in_source", "SELECT 1", interval=H6)
- self._node(source, moved)
- tier_id = tier_schedule_id(str(target.id), H6)
-
- target_listings = iter([set(), {tier_id}])
-
- def fake_listing(dag_id):
- if dag_id == str(target.id):
- return next(target_listings)
- return set()
-
- with (
- _temporal_boundary(schedules_by_dag={str(target.id): [tier_id]}) as mocks,
- mock.patch(f"{COMMAND}.list_existing_schedule_ids", side_effect=fake_listing),
- ):
- self._run(apply=True)
-
- moved.refresh_from_db()
- self.assertIsNone(moved.sync_frequency_interval)
- swept = {call.kwargs["schedule_id"] for call in mocks.v1_delete.call_args_list}
- self.assertEqual(swept, {str(moved.id)})
- mocks.tier_update.assert_called_once()
- self.assertEqual(get_declared_target(Node.objects.get(dag=target, saved_query=moved)), H6)
-
- def test_failed_move_keeps_source_dag_and_raises(self):
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- good = self._query("good_view", "SELECT 1")
- bad = self._query("bad_view", "SELECT * FROM nonexistent_table_xyz")
- self._node(source, good)
- self._node(source, bad)
-
- with _temporal_boundary() as mocks:
- with self.assertRaisesRegex(CommandError, "incomplete"):
- self._run(apply=True)
-
- # the DAG survives so the failed query's only node is not destroyed; the successful move
- # remains in the target as a duplicate a re-run classifies as DROP
- self.assertTrue(DAG.objects.filter(id=source.id).exists())
- self.assertTrue(Node.objects.filter(dag=source, saved_query=bad).exists())
- self.assertTrue(Node.objects.filter(dag=target, saved_query=good).exists())
- mocks.v2_delete.assert_not_called()
-
- def test_adopt_unresolvable_lands_broken_query_edge_less_and_deletes_source_dag(self):
- # One broken query must not hold the team's consolidation hostage: with the flag, the
- # unresolvable query is adopted into the target with no edges and a marker, cadence intent
- # intact, while healthy queries move normally and the source DAG is deleted.
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- good = self._query("good_view", "SELECT * FROM events")
- bad = self._query("bad_view", "SELECT * FROM nonexistent_table_xyz")
- self._node(source, good)
- set_declared_target(self._node(source, bad), M15)
-
- with _temporal_boundary():
- output = self._run("--adopt-unresolvable", apply=True)
-
- self.assertFalse(DAG.objects.filter(id=source.id).exists())
- adopted = Node.objects.get(dag=target, saved_query=bad)
- self.assertFalse(Edge.objects.filter(dag=target, target=adopted).exists())
- self.assertIn("nonexistent_table_xyz", adopted.properties["system"]["degraded_sync"]["error"])
- self.assertEqual(get_declared_target(adopted), M15)
- good_node = Node.objects.get(dag=target, saved_query=good)
- events = Node.objects.get(dag=target, name="events", type=NodeType.TABLE)
- self.assertTrue(Edge.objects.filter(dag=target, source=events, target=good_node).exists())
- self.assertIn("adopted bad_view without edges", output)
-
- def test_adopted_node_unblocks_its_dependents_moves(self):
- # The reason the flag exists: one unresolvable query must not cascade into N failed
- # moves. blocked_view depends on a saved query that has no DAG node (an orphan) → it
- # resolves in HogQL but fails Node.DoesNotExist and is adopted edge-less → its dependent
- # resolves the adopted node in the target and moves with a real edge.
- target = DAG.get_or_create_default(self.team)
- self._query("orphan_parent", "SELECT 1")
- source = DAG.objects.create(team=self.team, name="posthog_team")
- blocked = self._query("blocked_view", "SELECT * FROM orphan_parent")
- dependent = self._query("dependent_view", "SELECT * FROM blocked_view")
- blocked_node = self._node(source, blocked)
- dependent_node = self._node(source, dependent)
- Edge.objects.create(team=self.team, dag=source, source=blocked_node, target=dependent_node)
-
- with _temporal_boundary():
- output = self._run("--adopt-unresolvable", apply=True)
-
- self.assertFalse(DAG.objects.filter(id=source.id).exists())
- adopted = Node.objects.get(dag=target, saved_query=blocked)
- self.assertFalse(Edge.objects.filter(dag=target, target=adopted).exists())
- self.assertIn("degraded_sync", adopted.properties["system"])
- moved = Node.objects.get(dag=target, saved_query=dependent)
- self.assertTrue(Edge.objects.filter(dag=target, source=adopted, target=moved).exists())
- self.assertIn("adopted blocked_view without edges", output)
- self.assertIn("moved dependent_view", output)
-
- def test_adopt_unresolvable_does_not_adopt_non_resolution_failures(self):
- # The flag exists for SQL that cannot resolve. A sync that fails for operational reasons
- # (DB blip, programming error) must stay a failed move — adopting it would delete the
- # source DAG and report success on a query a plain retry would have moved intact.
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- moved = self._query("only_in_source", "SELECT 1")
- self._node(source, moved)
-
- with _temporal_boundary():
- with mock.patch(f"{COMMAND}.sync_saved_query_to_dag", side_effect=RuntimeError("db down")):
- with self.assertRaisesRegex(CommandError, "incomplete"):
- self._run("--adopt-unresolvable", apply=True)
-
- self.assertTrue(DAG.objects.filter(id=source.id).exists())
- self.assertFalse(Node.objects.filter(dag=target, saved_query=moved).exists())
-
- def test_adoption_failure_rolls_back_the_node(self):
- # If the marker save fails after get_or_create committed the node, an unmarked edge-less
- # node would survive; the re-run would then classify the query as DROP instead of
- # retrying the move, stranding it without the marker forever. Create + mark must commit
- # or roll back together so the re-run sees a clean MOVE to retry.
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- bad = self._query("bad_view", "SELECT * FROM nonexistent_table_xyz")
- self._node(source, bad)
-
- real_save = Node.save
-
- def failing_marker_save(node, *args, **kwargs):
- if kwargs.get("update_fields") == ["properties"]:
- raise RuntimeError("connection lost")
- return real_save(node, *args, **kwargs)
-
- with _temporal_boundary():
- with mock.patch.object(Node, "save", autospec=True, side_effect=failing_marker_save):
- with self.assertRaisesRegex(CommandError, "incomplete"):
- self._run("--adopt-unresolvable", apply=True)
-
- self.assertTrue(DAG.objects.filter(id=source.id).exists())
- self.assertFalse(Node.objects.filter(dag=target, saved_query=bad).exists())
-
- def test_degraded_sync_marker_cleared_when_query_resolves_again(self):
- # A stale marker after the customer fixes their SQL would make the post-fix re-sync sweep
- # (and any dashboard keyed on the marker) report a healthy node as degraded forever.
- target = DAG.get_or_create_default(self.team)
- fixed = self._query("fixed_view", "SELECT * FROM events")
- node = Node.objects.create(
- team=self.team,
- dag=target,
- saved_query=fixed,
- type=NodeType.VIEW,
- properties={"system": {"degraded_sync": {"error": "Unknown table", "at": "2026-07-30T00:00:00+00:00"}}},
- )
-
- sync_saved_query_to_dag(fixed, dag=target, reconcile=False)
-
- node.refresh_from_db()
- self.assertNotIn("system", node.properties)
- events = Node.objects.get(dag=target, name="events", type=NodeType.TABLE)
- self.assertTrue(Edge.objects.filter(dag=target, source=events, target=node).exists())
-
- def test_schedule_teardown_failure_keeps_source_dag(self):
- # deleting the DAG row before its execute-dag schedules are gone would orphan the
- # schedules forever (the row is the only pointer to them)
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- moved = self._query("only_in_source", "SELECT 1")
- self._node(source, moved)
- schedules_by_dag = {str(target.id): [str(target.id)], str(source.id): [str(source.id)]}
-
- with _temporal_boundary(schedules_by_dag=schedules_by_dag, v2_teardown_fails=True):
- with self.assertRaisesRegex(CommandError, "incomplete"):
- self._run(apply=True)
-
- self.assertTrue(DAG.objects.filter(id=source.id).exists())
- self.assertTrue(Node.objects.filter(dag=target, saved_query=moved).exists())
-
- def test_dry_run_by_default_mutates_nothing(self):
- source = DAG.objects.create(team=self.team, name="posthog_team")
- target = DAG.get_or_create_default(self.team)
- shared = self._query("shared_mv", "SELECT 1", interval=H6)
- self._node(target, shared)
- self._node(source, shared)
- moved = self._query("only_in_source", "SELECT 1", interval=H6)
- self._node(source, moved)
-
- with _temporal_boundary() as mocks:
- output = self._run()
-
- self.assertIn("move 1, drop 1", output)
- self.assertIn("dry run", output)
- self.assertTrue(DAG.objects.filter(id=source.id).exists())
- self.assertFalse(Node.objects.filter(dag=target, saved_query=moved).exists())
- moved.refresh_from_db()
- self.assertEqual(moved.sync_frequency_interval, H6)
- mocks.v1_delete.assert_not_called()
- mocks.v2_delete.assert_not_called()
-
- def test_dry_run_reports_edge_impact(self):
- # An edge touching a moved node must be re-pointed to the target; an edge whose only
- # involved node is dropped is redundant against the target's existing edge.
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- shared = self._query("shared_mv", "SELECT 1")
- self._node(target, shared)
- dropped_node = self._node(source, shared)
- moved_node = self._node(source, self._query("only_in_source", "SELECT 1"))
- events = Node.objects.create(team=self.team, dag=source, name="events", type=NodeType.TABLE)
- Edge.objects.create(team=self.team, dag=source, source=events, target=moved_node)
- Edge.objects.create(team=self.team, dag=source, source=events, target=dropped_node)
-
- output = self._run()
-
- self.assertIn("edges to re-point (touch a moved node): 1", output)
- self.assertIn("edges to drop as redundant (only dropped nodes): 1", output)
- self.assertIn("edges to re-point: 1, edges to drop: 1", output)
-
- def test_duplicate_is_kept_when_the_claiming_dag_failed_to_move_it(self):
- # dag_a claims the shared query for MOVE and dag_b's copy is planned as a DROP. If dag_a's
- # move fails, dag_b's node is the last place that query's declared target exists — dropping
- # it and deleting dag_b would destroy the intent with only a "dropped duplicate" line.
- DAG.get_or_create_default(self.team)
- dag_a = DAG.objects.create(team=self.team, name="dag_a")
- dag_b = DAG.objects.create(team=self.team, name="dag_b")
- shared = self._query("shared_mv", "SELECT * FROM nonexistent_table_xyz")
- self._node(dag_a, shared)
- self._node(dag_b, shared)
-
- with _temporal_boundary():
- with self.assertRaisesRegex(CommandError, "incomplete"):
- self._run(apply=True)
-
- # dag_a could not move it, so dag_b's copy is the last node the query has anywhere
- self.assertTrue(DAG.objects.filter(id=dag_b.id).exists())
- self.assertTrue(Node.objects.filter(dag=dag_b, saved_query=shared).exists())
-
- def test_dag_default_cadence_seeds_a_moved_node_with_no_interval(self):
- # A query only ever scheduled by its DAG's default cadence has no interval of its own. The
- # finalize nulls every interval and then reconciles, so with no DAG-level fallback the moved
- # node arrives with nothing declared, gets no effective cadence, and is dropped from every
- # tier — it stops materializing with nothing left in Postgres or Temporal to show why.
- target = DAG.get_or_create_default(self.team)
- target.sync_frequency_interval = H6
- target.save()
- source = DAG.objects.create(team=self.team, name="posthog_team")
- moved = self._query("only_in_source", "SELECT 1") # no interval of its own
- self._node(source, moved)
-
- tier_id = tier_schedule_id(str(target.id), H6)
- with _temporal_boundary(schedules_by_dag={str(target.id): [tier_id]}):
- self._run(apply=True)
-
- moved_node = Node.objects.get(dag=target, saved_query=moved)
- self.assertEqual(get_declared_target(moved_node), H6)
-
- def test_verbose_lists_query_names(self):
- # --verbose is how an operator confirms *which* queries move before committing to --apply
- # on live data, so the per-name listing has to actually appear behind the flag.
- DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- self._node(source, self._query("only_in_source", "SELECT 1"))
-
- self.assertNotIn("only_in_source", self._run())
- self.assertIn("move (in dependency order): only_in_source", self._run("--verbose"))
-
- def test_conflicting_freshness_targets_flagged(self):
- # Same query in two DAGs with different declared cadences: only one node survives the merge,
- # so the dry run must surface both copies for a human to pick the winner.
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- shared = self._query("conflicted", "SELECT 1")
- set_declared_target(self._node(target, shared), H6)
- set_declared_target(self._node(source, shared), M15)
-
- output = self._run()
-
- self.assertIn("freshness-target conflicts: 1", output)
- self.assertIn("⚠ conflicted:", output)
- self.assertIn("6hour", output)
- self.assertIn("15min", output)
-
- @parameterized.expand([("both_agree", H6, H6), ("one_unset", H6, None), ("both_unset", None, None)])
- def test_non_conflicting_freshness_targets_not_flagged(self, _name, target_a, target_b):
- # Unset means "no opinion", so a lone declared target wins cleanly — not a conflict.
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- shared = self._query("agreeing", "SELECT 1")
- a = self._node(target, shared)
- b = self._node(source, shared)
- if target_a is not None:
- set_declared_target(a, target_a)
- if target_b is not None:
- set_declared_target(b, target_b)
-
- self.assertIn("freshness-target conflicts: 0", self._run())
-
- @parameterized.expand(
- [
- ("default_preferred", "default"),
- ("largest_without_default", "largest"),
- ("explicit_wins", "explicit"),
- ]
- )
- def test_target_selection(self, _name, kind):
- if kind == "largest":
- small = DAG.objects.create(team=self.team, name="small")
- large = DAG.objects.create(team=self.team, name="large")
- self._node(small, self._query("a", "SELECT 1"))
- self._node(large, self._query("b", "SELECT 1"))
- self._node(large, self._query("c", "SELECT 1"))
-
- output = self._run()
-
- self.assertIn(f"Merge target: large ({large.id})", output)
- self.assertIn("largest DAG by node count", output)
- return
-
- default = DAG.get_or_create_default(self.team)
- other = DAG.objects.create(team=self.team, name="posthog_team")
- self._node(other, self._query("a", "SELECT 1"))
-
- if kind == "explicit":
- output = self._run("--target-dag-id", str(other.id))
- self.assertIn(f"Merge target: posthog_team ({other.id})", output)
- self.assertIn("chosen via --target-dag-id", output)
- else:
- output = self._run()
- self.assertIn(f"Merge target: Default ({default.id})", output)
- self.assertIn("canonical 'Default' DAG", output)
-
- @parameterized.expand(
- [
- ("default_dag_exists", DEFAULT_DAG_NAME),
- ("only_a_differently_named_dag", "posthog_team"),
- ("no_user_dag_yet", None),
- ]
- )
- def test_managed_dag_folds_into_the_default_dag(self, _name, user_dag_name):
- managed = DAG.get_or_create_revenue_analytics(self.team)
- managed_query = self._query("stripe.prod.mrr_revenue_view", "SELECT 1")
- managed_node = self._node(managed, managed_query)
- set_declared_target(managed_node, H12)
- if user_dag_name is not None:
- default = DAG.objects.create(team=self.team, name=user_dag_name)
- reader = self._node(default, self._query("reads_mrr", "SELECT 1"))
- proxy = Node.objects.create(
- team=self.team,
- dag=default,
- name=managed_query.name,
- type=NodeType.TABLE,
- properties={"origin": "cross_dag_view", "saved_query_id": str(managed_query.id)},
- )
- Edge.objects.create(team=self.team, dag=default, source=proxy, target=reader)
-
- with _temporal_boundary() as mocks:
- self._run(apply=True)
-
- target = DAG.objects.get(team=self.team)
- self.assertEqual(target.name, DEFAULT_DAG_NAME)
- node = Node.objects.get(saved_query=managed_query)
- self.assertEqual((node.id, node.dag_id), (managed_node.id, target.id))
- self.assertEqual(get_declared_target(node), H12)
- self.assertEqual(mocks.v2_delete.call_args.args, (str(managed.id),))
- self.assertEqual(mocks.tier_create.call_args.kwargs["id"], tier_schedule_id(str(target.id), H12))
- if user_dag_name is not None:
- self.assertEqual(list(Edge.objects.filter(target=reader).values_list("source", flat=True)), [node.id])
- self.assertFalse(Node.objects.filter(properties__has_key="saved_query_id").exists())
-
- @parameterized.expand([("teardown_succeeds", False), ("teardown_fails", True)])
- def test_teams_with_managed_dag_halts_at_the_first_failing_team(self, _name, teardown_fails):
- other_team = Team.objects.create(organization=self.organization)
- managed_dags = []
- for team in sorted([self.team, other_team], key=lambda team: team.pk):
- managed = DAG.objects.create(team=team, name=REVENUE_ANALYTICS_DAG_NAME)
- managed_query = DataWarehouseSavedQuery.objects.create(
- name="mrr_revenue_view", team=team, query={"query": "SELECT 1", "kind": "HogQLQuery"}
- )
- Node.objects.create(team=team, dag=managed, saved_query=managed_query, type=NodeType.MAT_VIEW)
- managed_dags.append(managed)
-
- with _temporal_boundary(v2_teardown_fails=teardown_fails):
- if teardown_fails:
- with self.assertRaisesRegex(CommandError, "halted at team"):
- call_command("consolidate_dags", "--teams-with-managed-dag", "--apply", stdout=StringIO())
- else:
- call_command("consolidate_dags", "--teams-with-managed-dag", "--apply", stdout=StringIO())
-
- remaining = set(DAG.objects.filter(name=REVENUE_ANALYTICS_DAG_NAME).values_list("id", flat=True))
- self.assertEqual(remaining, {dag.id for dag in managed_dags} if teardown_fails else set())
-
- def test_a_running_materialization_keeps_the_managed_dag_and_its_schedules(self):
- DAG.get_or_create_default(self.team)
- managed = DAG.get_or_create_revenue_analytics(self.team)
- managed_query = self._query("stripe.prod.mrr_revenue_view", "SELECT 1")
- managed_node = self._node(managed, managed_query)
- DataModelingJob.objects.create(team=self.team, saved_query=managed_query, status=DataModelingJob.Status.RUNNING)
-
- with _temporal_boundary() as mocks:
- with self.assertRaisesRegex(CommandError, "incomplete"):
- self._run(apply=True)
-
- mocks.v2_delete.assert_not_called()
- self.assertEqual(Node.objects.get(saved_query=managed_query).dag_id, managed_node.dag_id)
-
- def test_a_managed_view_that_fails_to_resolve_stays_in_the_managed_dag_with_its_edges(self):
- DAG.get_or_create_default(self.team)
- managed = DAG.get_or_create_revenue_analytics(self.team)
- managed_query = self._query("stripe.prod.mrr_revenue_view", "SELECT * FROM nonexistent_table_xyz")
- managed_node = self._node(managed, managed_query)
- parent = Node.objects.create(team=self.team, dag=managed, name="events", type=NodeType.TABLE)
- Edge.objects.create(team=self.team, dag=managed, source=parent, target=managed_node)
-
- with _temporal_boundary():
- with self.assertRaisesRegex(CommandError, "incomplete"):
- self._run(apply=True)
-
- self.assertTrue(DAG.objects.filter(id=managed.id).exists())
- self.assertEqual(Node.objects.get(saved_query=managed_query).dag_id, managed.id)
- self.assertTrue(Edge.objects.filter(dag=managed, source=parent, target=managed_node).exists())
-
- def test_folding_into_an_unscheduled_dag_seeds_targets_from_intervals(self):
- managed = DAG.get_or_create_revenue_analytics(self.team)
- managed_query = self._query("stripe.prod.mrr_revenue_view", "SELECT 1", interval=H12)
- self._node(managed, managed_query)
-
- with _temporal_boundary() as mocks:
- self._run(apply=True)
-
- managed_query.refresh_from_db()
- self.assertIsNone(managed_query.sync_frequency_interval)
- self.assertEqual(get_declared_target(Node.objects.get(saved_query=managed_query)), H12)
- self.assertEqual(
- mocks.tier_create.call_args.kwargs["id"],
- tier_schedule_id(str(DAG.objects.get(team=self.team).id), H12),
- )
-
- def test_a_managed_view_lands_from_the_managed_dag_not_from_a_user_dag_copy(self):
- target = DAG.get_or_create_default(self.team)
- managed = DAG.get_or_create_revenue_analytics(self.team)
- viewset = DataWarehouseManagedViewSet.objects.create(
- team=self.team, kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS
- )
- managed_query = self._query("stripe.prod.mrr_revenue_view", "SELECT 1")
- unplaced_managed_query = self._query("stripe.prod.charge_revenue_view", "SELECT 1")
- for query in (managed_query, unplaced_managed_query):
- query.managed_viewset = viewset
- query.save(update_fields=["managed_viewset"])
- set_declared_target(self._node(managed, managed_query), H12)
- set_declared_target(self._node(target, managed_query), timedelta(days=1))
- source = DAG.objects.create(team=self.team, name="posthog_team")
- self._node(source, managed_query)
- self._node(source, unplaced_managed_query)
- moved = self._query("only_in_source", "SELECT 1")
- self._node(source, moved)
-
- with _temporal_boundary():
- self._run(apply=True)
-
- self.assertEqual(list(DAG.objects.filter(team=self.team).values_list("id", flat=True)), [target.id])
- managed_node = Node.objects.get(saved_query=managed_query)
- self.assertEqual(managed_node.dag_id, target.id)
- self.assertEqual(get_declared_target(managed_node), H12)
- self.assertTrue(Node.objects.filter(dag=target, saved_query=moved).exists())
- self.assertTrue(Node.objects.filter(dag=target, saved_query=unplaced_managed_query).exists())
-
- @parameterized.expand(
- [
- ("managed_target", "managed"),
- ("unknown_target", "unknown"),
- ("not_a_uuid", "garbage"),
- ]
- )
- def test_bad_target_dag_id_errors_before_any_change(self, _name, kind):
- DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- self._node(source, self._query("only_in_source", "SELECT 1"))
- managed = DAG.get_or_create_revenue_analytics(self.team)
- target_id = {"managed": str(managed.id), "unknown": str(uuid.uuid4()), "garbage": "not-a-uuid"}[kind]
-
- with _temporal_boundary():
- with self.assertRaises(CommandError):
- self._run("--target-dag-id", target_id, apply=True)
-
- self.assertTrue(DAG.objects.filter(id=source.id).exists())
-
- @parameterized.expand([("two_sources", 2), ("three_sources", 3)])
- def test_query_in_several_source_dags_lands_exactly_once(self, _name: str, source_count: int):
- # `saved_query_unique_within_team_dag` means the query can only land once: the first source
- # moves it, every later one must drop. Three sources exercises a drop against a target that
- # already holds the node, which two sources never reaches.
- target = DAG.get_or_create_default(self.team)
- shared = self._query("shared_everywhere", "SELECT 1")
- sources = [DAG.objects.create(team=self.team, name=f"dag_{i}") for i in range(source_count)]
- for source in sources:
- self._node(source, shared)
-
- with _temporal_boundary():
- output = self._run(apply=True)
-
- self.assertFalse(DAG.objects.filter(id__in=[dag.id for dag in sources]).exists())
- self.assertEqual(Node.objects.filter(saved_query=shared).count(), 1)
- self.assertTrue(Node.objects.filter(dag=target, saved_query=shared).exists())
- self.assertIn(f"moved: 1, dropped: {source_count - 1}", output)
-
- @parameterized.expand(
- [
- ("moved_node_inherits", "move", None, M15),
- ("dropped_copies_onto_unset_target", "drop", None, M15),
- ("target_declared_wins", "drop", H6, H6),
- ]
- )
- def test_declared_target_carried_from_source(self, _name, kind, target_declared, expected):
- target = DAG.get_or_create_default(self.team)
- source = DAG.objects.create(team=self.team, name="posthog_team")
- saved_query = self._query("carried", "SELECT 1")
- if kind == "drop":
- target_node = self._node(target, saved_query)
- if target_declared is not None:
- set_declared_target(target_node, target_declared)
- source_node = self._node(source, saved_query)
- set_declared_target(source_node, M15)
-
- with _temporal_boundary():
- self._run(apply=True)
-
- node = Node.objects.get(dag=target, saved_query=saved_query)
- self.assertEqual(get_declared_target(node), expected)
diff --git a/products/data_modeling/backend/test/test_preview_freshness_schedules.py b/products/data_modeling/backend/test/test_preview_freshness_schedules.py
index e4c979a9022d..df295dcc6d0d 100644
--- a/products/data_modeling/backend/test/test_preview_freshness_schedules.py
+++ b/products/data_modeling/backend/test/test_preview_freshness_schedules.py
@@ -9,9 +9,8 @@ from django.core.management import call_command
from products.data_modeling.backend.logic.node_frequency import set_declared_target
from products.data_modeling.backend.models.dag import DAG
-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
from products.data_modeling.backend.models.edge import Edge
-from products.data_modeling.backend.models.node import Node, NodeType
+from products.data_modeling.backend.models.node import NodeType
from products.data_modeling.backend.test.helpers import (
no_existing_schedules as _no_existing_schedules,
saved_query_node as _saved_query_node,
@@ -138,25 +137,3 @@ class TestPreviewFreshnessSchedules(BaseTest):
self.assertNotIn("Effective cadences", summary.getvalue())
self.assertIn("Effective cadences", verbose.getvalue())
self.assertIn("0:15:00", verbose.getvalue())
-
- def test_cross_dag_duplication_flags_unsafe_drop(self):
- # one saved query materialized by two DAGs is double-materialized; a query unique to a DAG
- # makes that DAG unsafe to drop, one whose queries all live elsewhere is safe
- shared = DataWarehouseSavedQuery.objects.create(
- name="shared_mv", team=self.team, query={"query": "SELECT 1", "kind": "HogQLQuery"}
- )
- default_dag = DAG.objects.create(team=self.team, name="Default")
- canonical_dag = DAG.objects.create(team=self.team, name="posthog_team")
- Node.objects.create(team=self.team, dag=default_dag, saved_query=shared, type=NodeType.MAT_VIEW)
- Node.objects.create(team=self.team, dag=canonical_dag, saved_query=shared, type=NodeType.MAT_VIEW)
- _saved_query_node(self.team, default_dag, "only_in_default", NodeType.MAT_VIEW)
-
- module = "products.data_modeling.backend.logic.schedule_reconcile"
- out = StringIO()
- with mock.patch(f"{module}.async_connect", new=mock.AsyncMock(return_value=_no_existing_schedules())):
- call_command("preview_freshness_schedules", "--team-id", str(self.team.pk), stdout=out)
-
- output = out.getvalue()
- self.assertIn("in >1 DAG (double-materialized): 1", output)
- self.assertIn("only in Default (unsafe to drop): 1", output)
- self.assertIn("only in posthog_team (safe to drop): 0", output)
diff --git a/products/data_modeling/backend/test/test_repoint_managed_view_readers.py b/products/data_modeling/backend/test/test_repoint_managed_view_readers.py
deleted file mode 100644
index 05e387455e8e..000000000000
--- a/products/data_modeling/backend/test/test_repoint_managed_view_readers.py
+++ /dev/null
@@ -1,88 +0,0 @@
-from io import StringIO
-
-from posthog.test.base import BaseTest
-from unittest.mock import patch
-
-from django.core.management import call_command
-
-from parameterized import parameterized
-
-from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag
-from products.data_modeling.backend.models import DataModelingJob, DataModelingJobStatus, Edge, Node
-from products.data_modeling.backend.models.dag import DAG
-from products.data_modeling.backend.models.datawarehouse_managed_viewset import DataWarehouseManagedViewSet
-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-from products.data_modeling.backend.models.node import NodeType
-from products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKind
-
-MANAGED_VIEWSETS_FLAG = "managed-viewsets"
-
-
-class TestRepointManagedViewReaders(BaseTest):
- def setUp(self):
- super().setUp()
- flag_patch = patch(
- "posthog.hogql.database.database.feature_enabled_or_false",
- side_effect=lambda flag, *args, **kwargs: flag == MANAGED_VIEWSETS_FLAG,
- )
- flag_patch.start()
- self.addCleanup(flag_patch.stop)
-
- def _managed_view(self, name: str, sql: str, viewset: DataWarehouseManagedViewSet) -> DataWarehouseSavedQuery:
- view = DataWarehouseSavedQuery.objects.create(
- name=name,
- team=self.team,
- query={"query": sql, "kind": "HogQLQuery"},
- columns={"event": "String"},
- managed_viewset=viewset,
- )
- sync_saved_query_to_dag(view, dag=DAG.get_or_create_revenue_analytics(self.team), allow_managed=True)
- return view
-
- @parameterized.expand(
- [
- ("idle", False, False),
- ("charges_materializing", True, False),
- ("reader_fails_to_sync", False, True),
- ]
- )
- def test_only_apply_moves_readers_to_the_reference_and_deletes_the_copies(
- self, _name: str, charges_materializing: bool, reader_fails: bool
- ):
- viewset = DataWarehouseManagedViewSet.objects.create(
- team=self.team, kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS
- )
- charges = self._managed_view("revenue_charges", "SELECT event FROM events", viewset)
- mrr = self._managed_view("revenue_mrr", "SELECT event FROM revenue_charges", viewset)
- managed_dag = DAG.get_or_create_revenue_analytics(self.team)
- managed_node_ids = set(Node.objects.filter(dag=managed_dag).values_list("id", flat=True))
-
- default_dag = DAG.get_or_create_default(self.team)
- charges_copy = Node.objects.create(team=self.team, dag=default_dag, saved_query=charges, type=NodeType.MAT_VIEW)
- mrr_copy = Node.objects.create(team=self.team, dag=default_dag, saved_query=mrr, type=NodeType.MAT_VIEW)
- Edge.objects.create(team=self.team, dag=default_dag, source=charges_copy, target=mrr_copy)
- user_view = DataWarehouseSavedQuery.objects.create(
- name="user_view",
- team=self.team,
- query={"query": "" if reader_fails else "SELECT * FROM revenue_charges", "kind": "HogQLQuery"},
- )
- user_node = Node.objects.create(team=self.team, dag=default_dag, saved_query=user_view, type=NodeType.VIEW)
- Edge.objects.create(team=self.team, dag=default_dag, source=charges_copy, target=user_node)
- DataModelingJob.objects.create(team=self.team, saved_query=None, status=DataModelingJobStatus.RUNNING)
- if charges_materializing:
- DataModelingJob.objects.create(team=self.team, saved_query=charges, status=DataModelingJobStatus.RUNNING)
-
- call_command("repoint_managed_view_readers", team_id=self.team.pk, stdout=StringIO())
- self.assertEqual(Node.objects.filter(id__in=[charges_copy.id, mrr_copy.id]).count(), 2)
-
- call_command("repoint_managed_view_readers", team_id=self.team.pk, apply=True, stdout=StringIO())
-
- remaining = set(Node.objects.filter(id__in=[charges_copy.id, mrr_copy.id]).values_list("id", flat=True))
- self.assertEqual(remaining, {charges_copy.id} if charges_materializing or reader_fails else set())
- user_source = Edge.objects.get(target=user_node).source
- if reader_fails:
- self.assertEqual(user_source.id, charges_copy.id)
- else:
- self.assertEqual(user_source.type, NodeType.TABLE)
- self.assertEqual(user_source.properties["origin"], "cross_dag_view")
- self.assertEqual(set(Node.objects.filter(dag=managed_dag).values_list("id", flat=True)), managed_node_ids)