Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-modeling): drop the managed dag, proxies and node moves (#118772)

Thiago Salvatore86c17e979625
diff --git a/docs/internal/sql-editor-schema-tree.md b/docs/internal/sql-editor-schema-tree.md
index 8cf1e9b61d54..5a6b4fc5338e 100644
--- a/docs/internal/sql-editor-schema-tree.md
+++ b/docs/internal/sql-editor-schema-tree.md
@@ -60,8 +60,3 @@ Query validation and revision updates still run.
Use DELETE to remove a saved query so DAG cleanup, join cleanup, materialization cleanup, and name release run together.
Create does not upsert deleted rows.
A deleted row whose name was not released still reserves that name and needs separate cleanup.
-
-`dag_id` must identify a non-managed DAG in the same project; invalid or foreign IDs return 400.
-A DAG-only update moves the existing node and preserves its cadence and job history.
-Moves are refused when the view has dependents, belongs to a managed DAG, has nodes in multiple DAGs, or has a materialization running.
-Explicit `null` selects the default DAG.
diff --git a/posthog/temporal/data_modeling/activities/create_data_modeling_job.py b/posthog/temporal/data_modeling/activities/create_data_modeling_job.py
index 30a07157c9ce..3c5be23190fb 100644
--- a/posthog/temporal/data_modeling/activities/create_data_modeling_job.py
+++ b/posthog/temporal/data_modeling/activities/create_data_modeling_job.py
@@ -1,4 +1,3 @@
-import uuid
import dataclasses
from django.db import transaction
@@ -10,7 +9,6 @@ from temporalio import activity
from posthog.sync import database_sync_to_async_pool
from posthog.temporal.data_modeling.activities.utils import bind_data_modeling_log_context
-from products.data_modeling.backend.facade.api import lock_dag_placements_shared
from products.data_modeling.backend.facade.models import DataModelingJob, DataModelingJobEngine, Node
LOGGER = get_logger(__name__)
@@ -60,14 +58,6 @@ def _create_data_modeling_job(
inputs: CreateDataModelingJobInputs, workflow_id: str, workflow_run_id: str
) -> CreatedDataModelingJob:
with transaction.atomic():
- # A move to another DAG refuses while a job of this query is Running, and holds this DAG's
- # placement lock across that check. Taking it here, around the placement read and the
- # insert both, is what stops a job from being created against a placement the move has
- # already left. The rest of this materialization loads the node by team, node and DAG, so
- # such a job would find nothing, and that includes the activity which records the failure,
- # so the job row would stay Running with nothing left to close it. The lock is shared, so
- # job starts in one DAG do not queue behind each other.
- lock_dag_placements_shared(inputs.team_id, uuid.UUID(inputs.dag_id))
node = Node.objects.prefetch_related("saved_query").get(
id=inputs.node_id, team_id=inputs.team_id, dag_id=inputs.dag_id
)
diff --git a/posthog/temporal/tests/data_modeling/test_create_data_modeling_job_concurrency.py b/posthog/temporal/tests/data_modeling/test_create_data_modeling_job_concurrency.py
deleted file mode 100644
index a9f4ace06ff3..000000000000
--- a/posthog/temporal/tests/data_modeling/test_create_data_modeling_job_concurrency.py
+++ /dev/null
@@ -1,115 +0,0 @@
-import time
-import asyncio
-import threading
-from typing import Any
-
-import pytest
-import unittest.mock
-
-from django.db import connection, transaction
-
-from posthog.sync import database_sync_to_async
-from posthog.temporal.data_modeling.activities.create_data_modeling_job import (
- CreateDataModelingJobInputs,
- _create_data_modeling_job,
-)
-
-from products.data_modeling.backend.facade.api import NodeMoveError, lock_dag, move_saved_query_to_dag
-from products.data_modeling.backend.facade.models import DAG, DataModelingJob, DataModelingJobStatus, Node
-
-pytestmark = [pytest.mark.asyncio, pytest.mark.django_db(transaction=True)]
-
-BLOCKED_DEADLINE_SECONDS = 30
-
-
-def wait_until_another_backend_waits_on_a_lock() -> bool:
- """Whether some other connection is parked on a lock this test database can grant.
-
- Polled rather than slept on, so the wait ends as soon as the other thread reaches its lock
- instead of after a guessed interval. `pg_locks` is read rather than `pg_stat_activity`
- because the caller polls from inside an open transaction, and Postgres caches the backend
- status snapshot `pg_stat_activity` reports for the length of one. A transaction the waiter
- is queued behind is recorded with no database of its own, so both rows count.
- """
- deadline = time.monotonic() + BLOCKED_DEADLINE_SECONDS
- while time.monotonic() < deadline:
- with connection.cursor() as cursor:
- cursor.execute(
- "SELECT count(*) FROM pg_locks WHERE NOT granted AND pid <> pg_backend_pid() "
- "AND (database IS NULL OR database = (SELECT oid FROM pg_database WHERE datname = current_database()))"
- )
- if cursor.fetchone()[0] > 0:
- return True
- time.sleep(0.05)
- return False
-
-
-async def test_a_move_cannot_land_between_a_job_reading_a_placement_and_inserting_it(ateam, adag, anode, asaved_query):
- destination = await database_sync_to_async(DAG.objects.create)(team=ateam, name="destination")
- refusals: list[str] = []
-
- def move_on_another_connection() -> None:
- # A thread gets its own connection, so this move commits instead of joining the job's
- # transaction and rolling back with it.
- try:
- move_saved_query_to_dag(ateam.pk, asaved_query.id, destination.id)
- except NodeMoveError as e:
- refusals.append(e.reason)
- finally:
- connection.close()
-
- real_create = DataModelingJob.objects.create
- movers: list[threading.Thread] = []
-
- def create_after_a_move_tries_to_land(*args: Any, **kwargs: Any) -> Any:
- # Between the placement read and this insert is the window where a move whose Running
- # check ran before the job row existed could still commit, leaving the job pointing at a
- # DAG the node has left.
- mover = threading.Thread(target=move_on_another_connection)
- mover.start()
- movers.append(mover)
- assert wait_until_another_backend_waits_on_a_lock(), "the move was not held off by the job's DAG lock"
- return real_create(*args, **kwargs)
-
- inputs = CreateDataModelingJobInputs(team_id=ateam.pk, node_id=str(anode.id), dag_id=str(adag.id))
- with unittest.mock.patch.object(DataModelingJob.objects, "create", side_effect=create_after_a_move_tries_to_land):
- created = await _create_data_modeling_job(inputs, "test-workflow-id", "test-run-id")
-
- assert len(movers) == 1
- movers[0].join(timeout=BLOCKED_DEADLINE_SECONDS)
- assert not movers[0].is_alive()
-
- assert refusals == ["materializing"]
- node = await database_sync_to_async(Node.objects.get)(id=anode.id)
- assert node.dag_id == adag.id
- job = await database_sync_to_async(DataModelingJob.objects.get)(id=created.job_id)
- assert job.status == DataModelingJobStatus.RUNNING
-
-
-async def test_a_job_start_does_not_wait_behind_a_dag_edge_writer(ateam, adag, anode):
- held = threading.Event()
- release = threading.Event()
-
- def hold_the_dag_edge_lock_on_another_connection() -> None:
- try:
- with transaction.atomic():
- lock_dag(ateam.pk, adag.id)
- held.set()
- release.wait(timeout=BLOCKED_DEADLINE_SECONDS)
- finally:
- connection.close()
-
- writer = threading.Thread(target=hold_the_dag_edge_lock_on_another_connection)
- writer.start()
- try:
- assert held.wait(timeout=BLOCKED_DEADLINE_SECONDS)
- inputs = CreateDataModelingJobInputs(team_id=ateam.pk, node_id=str(anode.id), dag_id=str(adag.id))
- created = await asyncio.wait_for(
- _create_data_modeling_job(inputs, "test-workflow-id", "test-run-id"), timeout=BLOCKED_DEADLINE_SECONDS / 3
- )
- finally:
- release.set()
- writer.join(timeout=BLOCKED_DEADLINE_SECONDS)
-
- job = await database_sync_to_async(DataModelingJob.objects.get)(id=created.job_id)
- assert job.status == DataModelingJobStatus.RUNNING
diff --git a/products/data_modeling/backend/admin.py b/products/data_modeling/backend/admin.py
index e549fd6a3cf9..5cdd27364e85 100644
--- a/products/data_modeling/backend/admin.py
+++ b/products/data_modeling/backend/admin.py
@@ -1,6 +1,6 @@
from django.conf import settings
from django.contrib import admin
-from django.http import Http404
+from django.http import Http404, HttpRequest
from django.shortcuts import render
from django.urls import path, reverse
from django.utils.html import format_html
@@ -18,6 +18,15 @@ class DataModelingDAGAdmin(admin.ModelAdmin):
raw_id_fields = ("team", "created_by")
readonly_fields = ("id", "created_at", "updated_at", "tiers_link")
+ def has_add_permission(self, request: HttpRequest) -> bool:
+ return False
+
+ def has_change_permission(self, request: HttpRequest, obj: DAG | None = None) -> bool:
+ return False
+
+ def has_delete_permission(self, request: HttpRequest, obj: DAG | None = None) -> bool:
+ return False
+
def get_urls(self):
return [
path(
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index ffb61b24f486..092dfdfdac7c 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -18,15 +18,12 @@ _LAZY = {
"sync_metric_to_dag": "logic.metric_dag_sync",
"HasDependentsError": "logic.saved_query_dag_sync",
"MissingDagNodeError": "logic.saved_query_dag_sync",
- "NodeMoveError": "logic.saved_query_dag_sync",
"blocked_lineage_node_id": "logic.saved_query_dag_sync",
"check_saved_query_deletable": "logic.saved_query_dag_sync",
"delete_node_from_dag": "logic.saved_query_dag_sync",
"delete_node": "logic.saved_query_dag_sync",
"delete_orphaned_table_nodes": "logic.saved_query_dag_sync",
"lock_dag": "logic.saved_query_dag_sync",
- "lock_dag_placements_shared": "logic.saved_query_dag_sync",
- "move_saved_query_to_dag": "logic.saved_query_dag_sync",
"describe_dependents": "logic.saved_query_dag_sync",
"promote_view_nodes_to_matview": "logic.saved_query_dag_sync",
"sync_saved_query_to_dag": "logic.saved_query_dag_sync",
diff --git a/products/data_modeling/backend/facade/managed_viewset_hooks.py b/products/data_modeling/backend/facade/managed_viewset_hooks.py
index 34620f92335c..28ecf2abe479 100644
--- a/products/data_modeling/backend/facade/managed_viewset_hooks.py
+++ b/products/data_modeling/backend/facade/managed_viewset_hooks.py
@@ -37,8 +37,8 @@ class ProvidedView:
name: str
query: str
fields: dict[str, FieldOrTable]
- # Materialized views get a 12h sync schedule and a managed DAG node. A non-materialized view
- # (e.g. engineering analytics) is computed at query time — no schedule, no DAG, no S3 table.
+ # Materialized views get a 12h sync schedule. A non-materialized view (e.g. engineering
+ # analytics) is computed at query time — no schedule, no S3 table.
materialized: bool = True
diff --git a/products/data_modeling/backend/facade/models.py b/products/data_modeling/backend/facade/models.py
index 9d8a1faf00b2..471adc2d7d15 100644
--- a/products/data_modeling/backend/facade/models.py
+++ b/products/data_modeling/backend/facade/models.py
@@ -9,12 +9,7 @@ methods, so importing this adds nothing beyond the models Django loads at ``setu
"""
from products.data_modeling.backend.graph import Graph
-from products.data_modeling.backend.models.dag import (
- DAG,
- DEFAULT_DAG_NAME,
- RESERVED_DAG_NAMES,
- REVENUE_ANALYTICS_DAG_NAME,
-)
+from products.data_modeling.backend.models.dag import DAG, DEFAULT_DAG_NAME
from products.data_modeling.backend.models.data_modeling_job import (
DataModelingJob,
DataModelingJobEngine,
@@ -39,8 +34,6 @@ __all__ = [
"DAG",
"DEFAULT_DAG_NAME",
"NON_SCHEDULING_EDGES",
- "RESERVED_DAG_NAMES",
- "REVENUE_ANALYTICS_DAG_NAME",
"SAVED_QUERY_NODE_TYPES",
"DataModelingJob",
"DataModelingJobEngine",
diff --git a/products/data_modeling/backend/logic/saved_query_dag_sync.py b/products/data_modeling/backend/logic/saved_query_dag_sync.py
index 706ffb7eac3d..f498935aeb9e 100644
--- a/products/data_modeling/backend/logic/saved_query_dag_sync.py
+++ b/products/data_modeling/backend/logic/saved_query_dag_sync.py
@@ -17,8 +17,7 @@ from products.data_modeling.backend.facade.system_tables import DATA_MODELING_AL
from products.data_modeling.backend.logic.node_suspension import clear_suspension_if_query_changed
from products.data_modeling.backend.logic.saved_query_reads import saved_query_node_ids
from products.data_modeling.backend.logic.schedule_reconcile import maybe_reconcile_dag
-from products.data_modeling.backend.models.dag import DAG, REVENUE_ANALYTICS_DAG_NAME
-from products.data_modeling.backend.models.data_modeling_job import DataModelingJob, DataModelingJobStatus
+from products.data_modeling.backend.models.dag import DAG
from products.data_modeling.backend.models.edge import Edge
from products.data_modeling.backend.models.modeling import UnknownParentError, get_parents_from_model_query
from products.data_modeling.backend.models.node import Node, NodeType
@@ -65,35 +64,7 @@ def node_type_for(saved_query: "DataWarehouseSavedQuery") -> NodeType:
return NodeType.VIEW
-def get_dag_id(team_id: int) -> str:
- """Return the standard dag_id for a team."""
- return f"posthog_{team_id}"
-
-
-def _managed_cross_dag_reference(
- team: "Team", dag: DAG, dependency_name: str, saved_query: "DataWarehouseSavedQuery"
-) -> Node | None:
- """A table node in `dag` standing in for a saved-query parent whose node lives in a managed
- DAG (Revenue Analytics). Cross-dag edges are forbidden, so a same-dag reference is the only
- join available. Returns None when the parent is not in a managed DAG, so the caller resolves
- normally and a user's own extra DAG stays a loud failure rather than a silent orphan."""
- if not Node.objects.filter(team=team, saved_query=saved_query, dag__name=REVENUE_ANALYTICS_DAG_NAME).exists():
- return None
- node, _ = Node.objects.get_or_create(
- team=team,
- dag=dag,
- name=dependency_name,
- type=NodeType.TABLE,
- defaults={"properties": {"origin": "cross_dag_view", "saved_query_id": str(saved_query.id)}},
- )
- return node
-
-
def _saved_query_node(team: "Team", dag: DAG, dependency_name: str, saved_query: "DataWarehouseSavedQuery") -> Node:
- if not dag.is_managed:
- reference = _managed_cross_dag_reference(team, dag, dependency_name, saved_query)
- if reference is not None:
- return reference
node = Node.objects.filter(team=team, dag=dag, saved_query=saved_query).order_by("created_at").first()
if node is not None:
return node
@@ -189,12 +160,6 @@ def resolve_dependency_to_node(
return node
-class ManagedDAGError(Exception):
- """Raised when a user-initiated sync targets a system-managed DAG (e.g. Revenue Analytics)."""
-
- pass
-
-
def lock_dag(team_id: int, dag_id: UUID) -> None:
"""Serialize against every other writer of this DAG's edges.
@@ -205,26 +170,6 @@ def lock_dag(team_id: int, dag_id: UUID) -> None:
cursor.execute("SELECT pg_advisory_xact_lock(%s, hashtext(%s))", [team_id, str(dag_id)])
-def _placement_lock_key(dag_id: UUID) -> str:
- return f"placement:{dag_id}"
-
-
-def lock_dag_placements(team_id: int, dag_id: UUID) -> None:
- """Hold off every job start in this DAG, so a job cannot be created against a placement a
- move has already left behind.
-
- A separate key from `lock_dag`, so a job start does not queue behind edge writes and syncs.
- """
- with connection.cursor() as cursor:
- cursor.execute("SELECT pg_advisory_xact_lock(%s, hashtext(%s))", [team_id, _placement_lock_key(dag_id)])
-
-
-def lock_dag_placements_shared(team_id: int, dag_id: UUID) -> None:
- """Taken by each job start. Job starts do not block each other, only a move out of this DAG."""
- with connection.cursor() as cursor:
- cursor.execute("SELECT pg_advisory_xact_lock_shared(%s, hashtext(%s))", [team_id, _placement_lock_key(dag_id)])
-
-
def orphaned_table_nodes() -> QuerySet[Node]:
return Node.objects.filter(
type=NodeType.TABLE,
@@ -348,8 +293,6 @@ def lineage_database(team: "Team") -> Database:
def sync_saved_query_to_dag(
saved_query: "DataWarehouseSavedQuery",
extra_properties: dict | None = None, # TODO(andrew): remove this after backfill
- dag: DAG | None = None,
- allow_managed: bool = False,
reconcile: bool = True,
database: Database | None = None,
) -> Node | None:
@@ -365,23 +308,15 @@ def sync_saved_query_to_dag(
Args:
saved_query: The SavedQuery to sync to the DAG
extra_properties: Optional dict of properties to merge into created nodes and edges
- dag: Optional DAG to use. If not provided, uses the default team DAG.
- allow_managed: Whether placement into a system-managed DAG is permitted. Only the
- internal managed-viewset sync passes this; user-initiated callers must not, so a
- same-team user can't insert nodes/edges into a managed DAG via the saved-query API.
database: An optional prebuilt database to reuse for dependency resolution.
Returns the Node for the SavedQuery, or None if query parsing fails.
Raises QueryError or CycleDetectionError if the query would create an invalid DAG.
- Raises ManagedDAGError if dag is system-managed and allow_managed is False.
"""
extra_properties = extra_properties or {}
team = saved_query.team
- if dag is None:
- dag = DAG.get_or_create_default(team)
- if dag.is_managed and not allow_managed:
- raise ManagedDAGError(f"Cannot sync saved query into system-managed DAG: dag_id={dag.id}")
+ dag = DAG.get_or_create_default(team)
model_query = saved_query.query.get("query") if saved_query.query else None
if not model_query:
raise ValueError(f"DataWarehouseSavedQuery has no query: saved_query_id={saved_query.id}")
@@ -430,12 +365,12 @@ def sync_saved_query_to_dag(
return target
-def adopt_saved_query_without_edges(team_id: int, saved_query_id: UUID, error: str, dag_id: UUID | None = None) -> None:
+def adopt_saved_query_without_edges(team_id: int, saved_query_id: UUID, error: str) -> None:
"""Give a saved query whose lineage did not sync a node marked sync-failed, without rebuilding its edges."""
from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
saved_query = DataWarehouseSavedQuery.objects.select_related("team").get(team_id=team_id, id=saved_query_id)
- dag = DAG.objects.get(team_id=team_id, id=dag_id) if dag_id else DAG.get_or_create_default(saved_query.team)
+ dag = DAG.get_or_create_default(saved_query.team)
with transaction.atomic():
node, _ = Node.objects.get_or_create(
team_id=team_id,
@@ -642,124 +577,6 @@ def delete_node_from_dag(saved_query: "DataWarehouseSavedQuery") -> None:
maybe_reconcile_dag(dag)
-NodeMoveRefusal = Literal["multiple_placements", "blocked", "materializing", "moved"]
-
-
-class NodeMoveError(Exception):
- """Raised when a saved query's node cannot move to the requested DAG.
-
- Carries `reason` rather than prose, so a caller serving a user words the refusal itself.
- """
-
- def __init__(self, reason: NodeMoveRefusal) -> None:
- super().__init__(reason)
- self.reason = reason
-
-
-def move_saved_query_to_dag(team_id: int, saved_query_id: UUID, dag_id: UUID) -> None:
- """Place a saved query's node in the given DAG, keeping its frequency target and job history.
-
- Holds the source and destination DAGs for the whole check-and-move, in id order, for the
- reason `delete_node_from_dag` holds every DAG it touches: otherwise a sync can attach a
- dependent between the dependents check and the move, leaving an edge whose source sits in
- another DAG, which `Edge.save` treats as an invariant violation.
-
- A move is refused while a materialization runs. Its activities load the node by team, node
- and DAG, so the move makes the remaining ones stop finding it -- including the activity that
- records the failure, which leaves the job row Running with nothing to close it. Job creation
- takes the DAG's placement lock and re-reads the node in its DAG, so a job that starts between the
- check and the move cannot slip past.
-
- The saved-query row lock is held too, because a query edit picks the DAG to sync into from its
- own read of the node. Without it that edit can choose the DAG this move is leaving and then
- `get_or_create` a second node there, which later moves refuse as `multiple_placements`.
-
- That row lock is `FOR NO KEY UPDATE`, which is what makes it safe to hold alongside a DAG
- lock. `FOR UPDATE` conflicts with the `KEY SHARE` lock Postgres takes on this row for the
- foreign key of every job and node insert, so a materialization holding the placement lock and
- inserting its job row would deadlock against this transaction waiting for that same DAG.
- """
- from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-
- with transaction.atomic():
- saved_query = DataWarehouseSavedQuery.objects.select_for_update(no_key=True).get(
- team_id=team_id, id=saved_query_id
- )
- dag = DAG.objects.get(team_id=team_id, id=dag_id)
- placements = Node.objects.filter(team_id=team_id, saved_query=saved_query)
- held = {dag.id} | {node.dag_id for node in placements if node.dag_id is not None}
- for held_dag_id in sorted(held, key=str):
- lock_dag(team_id, held_dag_id)
- lock_dag_placements(team_id, held_dag_id)
-
- nodes = list(placements.select_related("dag"))
- if len(nodes) > 1:
- raise NodeMoveError("multiple_placements")
- if nodes and nodes[0].dag_id != dag.id:
- node = nodes[0]
- if node.dag_id not in held:
- # Another move landed between the read above and the locks, so the DAG this node
- # sits in now is not one of the ones being held.
- raise NodeMoveError("moved")
- if node.dag.is_managed or node.outgoing_edges.exists():
- raise NodeMoveError("blocked")
- if DataModelingJob.objects.filter(
- team_id=team_id, saved_query=saved_query, status=DataModelingJobStatus.RUNNING
- ).exists():
- raise NodeMoveError("materializing")
- previous_dag = node.dag
- # Rebuild parents in the destination without discarding the node's targets or job history.
- delete_orphaned_table_nodes(team_id, _delete_incoming_edges(team_id, node))
- node.dag = dag
- node.save(update_fields=["dag"])
- maybe_reconcile_dag(previous_dag)
- sync_saved_query_to_dag(saved_query, dag=dag)
-
-
-class ManagedNodeMaterializingError(Exception):
- """Raised when a managed view's node cannot move because a materialization of it is running."""
-
-
-def move_node_out_of_managed_dag(saved_query: "DataWarehouseSavedQuery", dag: DAG) -> DAG | None:
- """Move the saved query's node from the Revenue Analytics DAG into `dag`, returning the DAG it left."""
- with transaction.atomic():
- node = (
- Node.objects.filter(
- team_id=saved_query.team_id, saved_query=saved_query, dag__name=REVENUE_ANALYTICS_DAG_NAME
- )
- .select_related("dag")
- .first()
- )
- if node is None:
- return None
- managed_dag = node.dag
- for dag_id in sorted({managed_dag.id, dag.id}, key=str):
- lock_dag(saved_query.team_id, dag_id)
- lock_dag_placements(saved_query.team_id, dag_id)
- # A concurrent call may have moved the node between the read above and the locks. Acting on
- # the stale row would take the existence check below for a legacy copy and delete the node.
- node = (
- Node.objects.filter(team_id=saved_query.team_id, saved_query=saved_query, dag=managed_dag)
- .select_related("dag")
- .first()
- )
- if node is None:
- return None
- if DataModelingJob.objects.filter(
- team_id=saved_query.team_id, saved_query=saved_query, status=DataModelingJobStatus.RUNNING
- ).exists():
- raise ManagedNodeMaterializingError(saved_query.name)
- source_ids = _delete_incoming_edges(saved_query.team_id, node)
- Edge.objects.filter(team_id=saved_query.team_id, source=node).delete()
- if Node.objects.filter(team_id=saved_query.team_id, dag=dag, saved_query=saved_query).exists():
- node.delete()
- else:
- node.dag = dag
- node.save(update_fields=["dag"])
- delete_orphaned_table_nodes(saved_query.team_id, source_ids)
- return managed_dag
-
-
def update_node_type(saved_query: "DataWarehouseSavedQuery", type: NodeType) -> None:
"""Update a Node's type to MAT_VIEW when materialized."""
nodes = Node.objects.filter(team=saved_query.team, saved_query=saved_query).select_related("dag", "dag__team")
diff --git a/products/data_modeling/backend/logic/saved_query_reads.py b/products/data_modeling/backend/logic/saved_query_reads.py
index e2a28e5ac605..88cdb63ddace 100644
--- a/products/data_modeling/backend/logic/saved_query_reads.py
+++ b/products/data_modeling/backend/logic/saved_query_reads.py
@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING
from uuid import UUID
from django.conf import settings
-from django.db.models import Q
from ..facade.contracts import SavedQueryDefinition, SavedQuerySummary, UpstreamTableRef
from ..models.datawarehouse_saved_query import DataWarehouseSavedQuery
@@ -15,7 +14,6 @@ from .saved_query_freshness import saved_query_materialized_at
from .tier_run_report import MATERIALIZING_TYPES
POSTHOG_TABLE_ORIGIN = "posthog"
-PROXY_SAVED_QUERY_ID_PROPERTY = "saved_query_id"
WAREHOUSE_TABLE_ID_PROPERTY = "warehouse_table_id"
if TYPE_CHECKING:
@@ -282,16 +280,16 @@ def dependent_saved_query_ids(team_id: int, saved_query_ids: Collection[UUID]) -
def saved_query_node_ids(team_id: int, saved_query_id: UUID | str) -> set[str]:
- """The DAG nodes that stand for this saved query, cross-DAG proxy table nodes included."""
+ """The DAG nodes that stand for this saved query."""
return _node_ids_standing_for(team_id, {str(saved_query_id)})
def _node_ids_standing_for(team_id: int, saved_query_ids: set[str]) -> set[str]:
return {
str(node_id)
- for node_id in Node.objects.filter(team_id=team_id)
- .filter(Q(saved_query_id__in=saved_query_ids) | Q(properties__saved_query_id__in=saved_query_ids))
- .values_list("id", flat=True)
+ for node_id in Node.objects.filter(team_id=team_id, saved_query_id__in=saved_query_ids).values_list(
+ "id", flat=True
+ )
}
@@ -328,35 +326,18 @@ def upstream_table_refs(team_id: int, saved_query_id: UUID | str) -> frozenset[U
is_posthog_table=properties.get("origin") == POSTHOG_TABLE_ORIGIN,
)
for name, properties in _upstream_table_nodes(team_id, str(saved_query_id))
- if _proxied_saved_query_id(properties) is None
)
def _upstream_table_nodes(team_id: int, saved_query_id: str) -> list[tuple[str, dict[str, object]]]:
- tables_by_node_id: dict[str, tuple[str, dict[str, object]]] = {}
- walked_saved_query_ids: set[str] = set()
- pending_saved_query_ids = {saved_query_id}
- while pending_saved_query_ids:
- walked_saved_query_ids |= pending_saved_query_ids
- start_node_ids = _node_ids_standing_for(team_id, pending_saved_query_ids)
- reached = reachable_node_ids(team_id, start_node_ids, upstream=True, max_depth=None)
- for node_id, name, properties in Node.objects.filter(
- team_id=team_id, id__in=reached, type=NodeType.TABLE
- ).values_list("id", "name", "properties"):
- tables_by_node_id[str(node_id)] = (name, properties or {})
- pending_saved_query_ids = _proxied_saved_query_ids(tables_by_node_id.values()) - walked_saved_query_ids
- return list(tables_by_node_id.values())
-
-
-def _proxied_saved_query_ids(table_nodes: Iterable[tuple[str, dict[str, object]]]) -> set[str]:
- return {
- proxied_id for _, properties in table_nodes if (proxied_id := _proxied_saved_query_id(properties)) is not None
- }
-
-
-def _proxied_saved_query_id(properties: dict[str, object]) -> str | None:
- proxied_id = properties.get(PROXY_SAVED_QUERY_ID_PROPERTY)
- return str(proxied_id) if proxied_id else None
+ start_node_ids = _node_ids_standing_for(team_id, {saved_query_id})
+ reached = reachable_node_ids(team_id, start_node_ids, upstream=True, max_depth=None)
+ return [
+ (name, properties or {})
+ for name, properties in Node.objects.filter(team_id=team_id, id__in=reached, type=NodeType.TABLE).values_list(
+ "name", "properties"
+ )
+ ]
def _warehouse_table_id(properties: dict[str, object]) -> str | None:
diff --git a/products/data_modeling/backend/models/__init__.py b/products/data_modeling/backend/models/__init__.py
index c48017eab4b2..c743a00b2a22 100644
--- a/products/data_modeling/backend/models/__init__.py
+++ b/products/data_modeling/backend/models/__init__.py
@@ -1,4 +1,4 @@
-from .dag import DAG, DEFAULT_DAG_NAME, RESERVED_DAG_NAMES, REVENUE_ANALYTICS_DAG_NAME
+from .dag import DAG, DEFAULT_DAG_NAME
from .data_modeling_job import DataModelingJob, DataModelingJobEngine, DataModelingJobStatus
from .datawarehouse_managed_viewset import DataWarehouseManagedViewSet
from .datawarehouse_saved_query import DataWarehouseSavedQuery
@@ -13,8 +13,6 @@ from .node import Node, NodeType
__all__ = [
"DAG",
"DEFAULT_DAG_NAME",
- "RESERVED_DAG_NAMES",
- "REVENUE_ANALYTICS_DAG_NAME",
"CycleDetectionError",
"DAGMismatchError",
"DataModelingEdgeManager",
diff --git a/products/data_modeling/backend/models/dag.py b/products/data_modeling/backend/models/dag.py
index afec81adafa7..bad4eb56583e 100644
--- a/products/data_modeling/backend/models/dag.py
+++ b/products/data_modeling/backend/models/dag.py
@@ -11,10 +11,6 @@ if TYPE_CHECKING:
from posthog.models import Team
DEFAULT_DAG_NAME = "Default"
-REVENUE_ANALYTICS_DAG_NAME = "PostHog Revenue Analytics"
-
-# Names users may not claim via the API — they belong to system-controlled DAGs.
-RESERVED_DAG_NAMES = frozenset({DEFAULT_DAG_NAME, REVENUE_ANALYTICS_DAG_NAME})
class DAG(UUIDModel, CreatedMetaFields, UpdatedMetaFields):
@@ -33,20 +29,6 @@ class DAG(UUIDModel, CreatedMetaFields, UpdatedMetaFields):
dag, _ = cls.objects.get_or_create(team=team, name=DEFAULT_DAG_NAME)
return dag
- @classmethod
- def get_or_create_revenue_analytics(cls, team: Team) -> DAG:
- dag, _ = cls.objects.get_or_create(team=team, name=REVENUE_ANALYTICS_DAG_NAME)
- return dag
-
- @property
- def is_default(self) -> bool:
- return self.name == DEFAULT_DAG_NAME
-
- @property
- def is_managed(self) -> bool:
- """System-managed DAGs (e.g. Revenue Analytics) cannot be renamed, edited, or deleted by users."""
- return self.name == REVENUE_ANALYTICS_DAG_NAME
-
class Meta:
db_table = "posthog_datamodelingdag"
constraints = [
diff --git a/products/data_modeling/backend/models/datawarehouse_managed_viewset.py b/products/data_modeling/backend/models/datawarehouse_managed_viewset.py
index 863c3bedea4a..6d5c826f5807 100644
--- a/products/data_modeling/backend/models/datawarehouse_managed_viewset.py
+++ b/products/data_modeling/backend/models/datawarehouse_managed_viewset.py
@@ -76,11 +76,7 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
Materializes views by default.
"""
# Deferred: schedule_reconcile pulls in temporalio; a module-level import would drag it onto django.setup()
- from products.data_modeling.backend.logic.saved_query_dag_sync import ( # noqa: PLC0415
- ManagedNodeMaterializingError,
- move_node_out_of_managed_dag,
- sync_saved_query_to_dag,
- )
+ from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag # noqa: PLC0415
from products.data_modeling.backend.logic.schedule_reconcile import maybe_reconcile_dag # noqa: PLC0415
provider = get_expected_views_provider(self.kind)
@@ -200,7 +196,6 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
# can therefore race on node/edge placement; that is tolerated for now, pending a pooling-safe
# guard. The transaction-scoped lock above still serializes the saved-query writes.
dag = DAG.get_or_create_default(self.team)
- left_managed_dags: set[DAG] = set()
if saved_queries_to_sync:
dag_database = Database.create_for(
team=self.team,
@@ -209,17 +204,8 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
)
for saved_query in saved_queries_to_sync:
try:
- left_dag = move_node_out_of_managed_dag(saved_query, dag)
- if left_dag is not None:
- left_managed_dags.add(left_dag)
# reconcile once after both loops, not once per view
- sync_saved_query_to_dag(saved_query, dag=dag, reconcile=False, database=dag_database)
- except ManagedNodeMaterializingError:
- logger.info(
- "deferred_moving_managed_view_while_materializing",
- team_id=self.team_id,
- view_name=saved_query.name,
- )
+ sync_saved_query_to_dag(saved_query, reconcile=False, database=dag_database)
except Exception as e:
capture_exception(e, {"managed_viewset_id": self.id, "view_name": saved_query.name})
logger.warning(
@@ -243,8 +229,6 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
if saved_queries_to_sync:
maybe_reconcile_dag(dag)
- for left_dag in left_managed_dags:
- maybe_reconcile_dag(left_dag)
views_deleted = 0
for orphaned_view in orphaned_views_to_revert:
diff --git a/products/data_modeling/backend/test/test_dag_admin.py b/products/data_modeling/backend/test/test_dag_admin.py
index cae8f243b7bf..47ab30445306 100644
--- a/products/data_modeling/backend/test/test_dag_admin.py
+++ b/products/data_modeling/backend/test/test_dag_admin.py
@@ -83,11 +83,19 @@ class TestDataModelingDAGAdminTiers(BaseTest):
assert response.status_code == 404
- def test_changelist_and_add_form_render(self):
- # both call into the display helpers that reverse other admin URLs, and the add form is the
- # only caller of tiers_link with an unsaved object
+ def test_admin_shows_dags_but_cannot_create_change_or_delete_them(self):
assert self.client.get(reverse("admin:data_modeling_dag_changelist")).status_code == 200
- assert self.client.get(reverse("admin:data_modeling_dag_add")).status_code == 200
+ assert self.client.get(reverse("admin:data_modeling_dag_change", args=[self.dag.id])).status_code == 200
+ assert self.client.get(reverse("admin:data_modeling_dag_add")).status_code == 403
+ assert (
+ self.client.post(
+ reverse("admin:data_modeling_dag_change", args=[self.dag.id]), {"name": "Renamed"}
+ ).status_code
+ == 403
+ )
+ assert self.client.post(reverse("admin:data_modeling_dag_delete", args=[self.dag.id])).status_code == 403
+ self.dag.refresh_from_db()
+ assert self.dag.name == "Default"
def test_tiers_page_requires_staff(self):
self.user.is_staff = False
diff --git a/products/data_modeling/backend/test/test_metric_dag_sync.py b/products/data_modeling/backend/test/test_metric_dag_sync.py
index 9378be0db40a..2f674d4923e4 100644
--- a/products/data_modeling/backend/test/test_metric_dag_sync.py
+++ b/products/data_modeling/backend/test/test_metric_dag_sync.py
@@ -14,7 +14,7 @@ from products.data_modeling.backend.logic.metric_dag_sync import (
)
from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag
from products.data_modeling.backend.models import Edge, Node
-from products.data_modeling.backend.models.dag import DAG, DEFAULT_DAG_NAME
+from products.data_modeling.backend.models.dag import DEFAULT_DAG_NAME
from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
from products.data_modeling.backend.models.node import DEGRADED_SYNC_KEY, UNRESOLVED_DEPENDENCIES_KEY, NodeType
@@ -99,21 +99,6 @@ class TestSyncMetricToDag(BaseTest):
self.assertNotIn("system", self._node().properties)
- def test_a_metric_over_a_managed_view_reaches_it_through_a_proxy_table_node(self):
- managed_dag = DAG.get_or_create_revenue_analytics(self.team)
- managed_query = DataWarehouseSavedQuery.objects.create(
- name="revenue_view", team=self.team, query={"query": "SELECT * FROM events", "kind": "HogQLQuery"}
- )
- sync_saved_query_to_dag(managed_query, dag=managed_dag, allow_managed=True)
-
- sync_metric_to_dag(self.team, self.metric_id, "mrr", ["revenue_view"])
-
- default_dag = DAG.objects.get(team=self.team, name=DEFAULT_DAG_NAME)
- proxy = Node.objects.get(team=self.team, dag=default_dag, name="revenue_view")
- self.assertEqual(proxy.type, NodeType.TABLE)
- self.assertEqual(proxy.properties["origin"], "cross_dag_view")
- self.assertTrue(Edge.objects.filter(source=proxy, target=self._node()).exists())
-
def test_a_metric_can_share_a_name_with_the_table_it_reads(self):
sync_metric_to_dag(self.team, self.metric_id, "events", ["events"])
diff --git a/products/data_modeling/backend/test/test_saved_query_dag_sync.py b/products/data_modeling/backend/test/test_saved_query_dag_sync.py
index f9063b2ebd22..6224914dcf79 100644
--- a/products/data_modeling/backend/test/test_saved_query_dag_sync.py
+++ b/products/data_modeling/backend/test/test_saved_query_dag_sync.py
@@ -4,8 +4,6 @@ import pytest
from posthog.test.base import BaseTest
from unittest import mock
-from django.test import SimpleTestCase
-
from parameterized import parameterized
from posthog.hogql.database.database import Database
@@ -14,13 +12,10 @@ from posthog.hogql.errors import QueryError, TableAccessDeniedError
from products.data_modeling.backend.facade.system_tables import DATA_MODELING_ALLOWED_SYSTEM_TABLES
from products.data_modeling.backend.logic.saved_query_dag_sync import (
HasDependentsError,
- ManagedDAGError,
blocked_lineage_node_id,
delete_node_from_dag,
describe_dependents,
- get_dag_id,
get_dependent_saved_queries,
- move_saved_query_to_dag,
sync_saved_query_to_dag,
update_node_type,
)
@@ -32,12 +27,6 @@ from products.data_modeling.backend.models.node import NodeType
from products.warehouse_sources.backend.facade.models import DataWarehouseTable
-class TestGetDagId(SimpleTestCase):
- def test_get_dag_id_returns_expected_format(self):
- self.assertEqual(get_dag_id(123), "posthog_123")
- self.assertEqual(get_dag_id(1), "posthog_1")
-
-
@pytest.mark.django_db
class TestSyncSavedQueryToDag(BaseTest):
@parameterized.expand(
@@ -81,32 +70,6 @@ class TestSyncSavedQueryToDag(BaseTest):
dag = DAG.objects.get(team=self.team, name=DEFAULT_DAG_NAME)
self.assertEqual(dag.name, DEFAULT_DAG_NAME)
- def test_sync_rejects_managed_dag_for_user_initiated_call(self):
- managed_dag = DAG.get_or_create_revenue_analytics(self.team)
- saved_query = DataWarehouseSavedQuery.objects.create(
- name="sneaky_view",
- team=self.team,
- query={"query": "SELECT 1", "kind": "HogQLQuery"},
- )
-
- with self.assertRaises(ManagedDAGError):
- sync_saved_query_to_dag(saved_query, dag=managed_dag)
-
- self.assertFalse(Node.objects.filter(team=self.team, dag=managed_dag).exists())
-
- def test_sync_allows_managed_dag_for_internal_call(self):
- managed_dag = DAG.get_or_create_revenue_analytics(self.team)
- saved_query = DataWarehouseSavedQuery.objects.create(
- name="managed_view",
- team=self.team,
- query={"query": "SELECT 1", "kind": "HogQLQuery"},
- )
-
- node = sync_saved_query_to_dag(saved_query, dag=managed_dag, allow_managed=True)
-
- assert node is not None
- self.assertEqual(node.dag_id, managed_dag.id)
-
def test_sync_reuses_existing_dag_model(self):
existing_dag = DAG.objects.create(team=self.team, name=DEFAULT_DAG_NAME)
@@ -348,65 +311,21 @@ class TestSyncSavedQueryToDag(BaseTest):
edge = Edge.objects.filter(source=upstream_node, target=downstream_node).first()
self.assertIsNotNone(edge)
- @parameterized.expand(
- [
- ("view", False, False),
- ("view_with_stale_default_copy", False, True),
- ("matview_with_stale_default_copy", True, True),
- ]
- )
- def test_sync_references_parent_in_managed_dag(self, _name: str, materialized: bool, stale_copy: bool):
- managed_dag = DAG.get_or_create_revenue_analytics(self.team)
- upstream_query = DataWarehouseSavedQuery.objects.create(
- name="upstream_view",
- team=self.team,
- query={"query": "SELECT event FROM events", "kind": "HogQLQuery"},
- columns={"event": "String"},
- )
- if materialized:
- upstream_query.table = DataWarehouseTable.objects.create(
- team=self.team, name="upstream_view", format="Parquet", columns={"event": "String"}
- )
- upstream_query.is_materialized = True
- upstream_query.save(update_fields=["table", "is_materialized"])
- sync_saved_query_to_dag(upstream_query, dag=managed_dag, allow_managed=True)
- default_dag = DAG.get_or_create_default(self.team)
- if stale_copy:
- Node.objects.create(team=self.team, dag=default_dag, saved_query=upstream_query, type=NodeType.MAT_VIEW)
-
- downstream_query = DataWarehouseSavedQuery.objects.create(
- name="downstream_view",
- team=self.team,
- query={"query": "SELECT * FROM upstream_view", "kind": "HogQLQuery"},
- )
- downstream_node = sync_saved_query_to_dag(downstream_query)
-
- reference = Node.objects.get(team=self.team, dag=default_dag, name="upstream_view", saved_query__isnull=True)
- self.assertEqual(reference.type, NodeType.TABLE)
- self.assertIsNone(reference.saved_query)
- self.assertTrue(Edge.objects.filter(source=reference, target=downstream_node).exists())
-
- def test_sync_does_not_reference_parent_in_non_managed_dag(self):
- other_dag = DAG.objects.create(team=self.team, name="user_made_dag")
- upstream_query = DataWarehouseSavedQuery.objects.create(
+ def test_sync_rolls_back_its_node_when_a_parent_view_has_no_node(self):
+ DataWarehouseSavedQuery.objects.create(
name="upstream_view",
team=self.team,
query={"query": "SELECT * FROM events", "kind": "HogQLQuery"},
)
- sync_saved_query_to_dag(upstream_query, dag=other_dag)
-
downstream_query = DataWarehouseSavedQuery.objects.create(
name="downstream_view",
team=self.team,
query={"query": "SELECT * FROM upstream_view", "kind": "HogQLQuery"},
)
+
with self.assertRaises(Node.DoesNotExist):
sync_saved_query_to_dag(downstream_query)
- default_dag = DAG.objects.get(team=self.team, name=DEFAULT_DAG_NAME)
- self.assertFalse(Node.objects.filter(team=self.team, dag=default_dag, name="upstream_view").exists())
- # the sync got as far as creating downstream_view's node before resolution failed, so the
- # failure must take that node with it rather than leave one with no edges
self.assertFalse(Node.objects.filter(saved_query=downstream_query).exists())
def test_sync_raises_on_cycle(self):
@@ -540,35 +459,6 @@ class TestDeleteNodeFromDag(BaseTest):
events_node = Node.objects.get(team=self.team, name="events", type=NodeType.TABLE)
self.assertTrue(Edge.objects.filter(source=events_node, target=remaining_node).exists())
- @parameterized.expand([("last_reader_moves", False, False), ("another_reader_stays", True, True)])
- def test_move_deletes_the_old_table_node_only_when_nothing_else_reads_it(
- self, _name: str, another_reader_stays: bool, old_table_node_kept: bool
- ):
- moved_view = DataWarehouseSavedQuery.objects.create(
- name="moved_view",
- team=self.team,
- query={"query": "SELECT * FROM events", "kind": "HogQLQuery"},
- )
- moved_node = sync_saved_query_to_dag(moved_view)
- assert moved_node is not None
- if another_reader_stays:
- sync_saved_query_to_dag(
- DataWarehouseSavedQuery.objects.create(
- name="staying_view",
- team=self.team,
- query={"query": "SELECT * FROM events", "kind": "HogQLQuery"},
- )
- )
- destination = DAG.objects.create(team=self.team, name="destination")
-
- move_saved_query_to_dag(self.team.pk, moved_view.id, destination.id)
-
- self.assertEqual(
- Node.objects.filter(team=self.team, dag=moved_node.dag, name="events", type=NodeType.TABLE).exists(),
- old_table_node_kept,
- )
- self.assertTrue(Node.objects.filter(team=self.team, dag=destination, name="events").exists())
-
def test_delete_handles_nonexistent_node(self):
saved_query = DataWarehouseSavedQuery.objects.create(
name="test_view",
@@ -629,61 +519,6 @@ class TestDeleteNodeFromDag(BaseTest):
str(upstream_node.id),
)
- def test_delete_finds_a_metric_hanging_off_any_of_the_querys_nodes(self):
- upstream = DataWarehouseSavedQuery.objects.create(
- name="upstream_view",
- team=self.team,
- query={"query": "SELECT * FROM events", "kind": "HogQLQuery"},
- )
- managed_dag = DAG.get_or_create_revenue_analytics(self.team)
- managed_node = sync_saved_query_to_dag(upstream, dag=managed_dag, allow_managed=True)
- default_node = sync_saved_query_to_dag(upstream)
- assert managed_node is not None and default_node is not None
- metric = Node.objects.create(
- team=self.team,
- dag=default_node.dag,
- name="weekly_active_accounts",
- type=NodeType.METRIC,
- metric_id=uuid4(),
- )
- Edge.objects.create(team=self.team, dag=default_node.dag, source=default_node, target=metric)
- self.assertNotEqual(managed_node.id, default_node.id)
-
- with self.assertRaises(HasDependentsError) as context:
- delete_node_from_dag(upstream)
-
- self.assertIn(
- "weekly_active_accounts (metric)", describe_dependents(upstream.name, context.exception.dependents)
- )
- self.assertEqual(
- blocked_lineage_node_id(context.exception.dependents, context.exception.fallback_node_id),
- str(default_node.id),
- )
-
- def test_delete_finds_a_dependent_view_hanging_off_any_of_the_querys_nodes(self):
- upstream = DataWarehouseSavedQuery.objects.create(
- name="upstream_view",
- team=self.team,
- query={"query": "SELECT * FROM events", "kind": "HogQLQuery"},
- )
- managed_dag = DAG.get_or_create_revenue_analytics(self.team)
- managed_node = sync_saved_query_to_dag(upstream, dag=managed_dag, allow_managed=True)
- default_node = sync_saved_query_to_dag(upstream)
- assert managed_node is not None and default_node is not None
- self.assertNotEqual(managed_node.id, default_node.id)
- # the dependent reads upstream_view in the default DAG, so its edge hangs off the newer node
- downstream = DataWarehouseSavedQuery.objects.create(
- name="downstream_view",
- team=self.team,
- query={"query": "SELECT * FROM upstream_view", "kind": "HogQLQuery"},
- )
- sync_saved_query_to_dag(downstream)
-
- with self.assertRaises(HasDependentsError) as context:
- delete_node_from_dag(upstream)
-
- self.assertIn("downstream_view (view)", describe_dependents(upstream.name, context.exception.dependents))
-
def test_delete_succeeds_when_no_dependents(self):
upstream = DataWarehouseSavedQuery.objects.create(
name="upstream_view",
@@ -751,27 +586,6 @@ class TestGetDependents(BaseTest):
dependents = get_dependent_saved_queries(upstream)
self.assertEqual(dependents, [])
- def test_get_dependents_reports_a_dependent_in_several_dags_once(self):
- upstream = DataWarehouseSavedQuery.objects.create(
- name="upstream_view",
- team=self.team,
- query={"query": "SELECT * FROM events", "kind": "HogQLQuery"},
- )
- managed_dag = DAG.get_or_create_revenue_analytics(self.team)
- sync_saved_query_to_dag(upstream, dag=managed_dag, allow_managed=True)
- sync_saved_query_to_dag(upstream)
- downstream = DataWarehouseSavedQuery.objects.create(
- name="downstream_view",
- team=self.team,
- query={"query": "SELECT * FROM upstream_view", "kind": "HogQLQuery"},
- )
- sync_saved_query_to_dag(downstream, dag=managed_dag, allow_managed=True)
- sync_saved_query_to_dag(downstream)
-
- dependents = get_dependent_saved_queries(upstream)
-
- self.assertEqual([d.name for d in dependents], ["downstream_view"])
-
def test_get_dependents_dedupes_and_orders_across_dags(self):
upstream = DataWarehouseSavedQuery.objects.create(
name="upstream_view",
diff --git a/products/data_modeling/backend/test/test_saved_query_reads.py b/products/data_modeling/backend/test/test_saved_query_reads.py
index a84a8f93cb6b..c59dbc8f7d0d 100644
--- a/products/data_modeling/backend/test/test_saved_query_reads.py
+++ b/products/data_modeling/backend/test/test_saved_query_reads.py
@@ -149,20 +149,13 @@ class TestSavedQueryReads(BaseTest):
near.id: frozenset(),
}
- def test_upstream_table_refs_walks_through_views_and_other_dags_to_tables_only(self) -> None:
+ def test_upstream_table_refs_walks_through_views_to_tables_only(self) -> None:
dag = DAG.objects.create(team=self.team, name="Default")
- other_dag = DAG.objects.create(team=self.team, name="Finance")
- invoices = table_node(self.team, other_dag, "invoices", {"origin": POSTHOG_TABLE_ORIGIN})
- revenue = saved_query_node(self.team, other_dag, "revenue", NodeType.VIEW)
- Edge.objects.create(team=self.team, dag=other_dag, source=invoices, target=revenue)
warehouse_table_id = str(uuid4())
charges = table_node(
self.team, dag, "stripe_charges", {"origin": "warehouse", "warehouse_table_id": warehouse_table_id}
)
events = table_node(self.team, dag, "events", {"origin": POSTHOG_TABLE_ORIGIN})
- proxy = table_node(
- self.team, dag, "revenue", {"origin": "cross_dag_view", "saved_query_id": str(revenue.saved_query_id)}
- )
persons = table_node(self.team, dag, "persons", {"origin": POSTHOG_TABLE_ORIGIN})
legacy = table_node(self.team, dag, "legacy_orders", {})
middle = saved_query_node(self.team, dag, "middle", NodeType.VIEW)
@@ -171,7 +164,6 @@ class TestSavedQueryReads(BaseTest):
for source, dependent in [
(charges, middle),
(events, middle),
- (proxy, middle),
(legacy, middle),
(middle, target),
(target, downstream),
@@ -183,7 +175,6 @@ class TestSavedQueryReads(BaseTest):
{
UpstreamTableRef(name="stripe_charges", warehouse_table_id=warehouse_table_id),
UpstreamTableRef(name="events", is_posthog_table=True),
- UpstreamTableRef(name="invoices", is_posthog_table=True),
UpstreamTableRef(name="legacy_orders"),
}
)
diff --git a/products/data_modeling/backend/tests/test_managed_viewset_providers.py b/products/data_modeling/backend/tests/test_managed_viewset_providers.py
index 59d3f6a1ce63..f93b8627b08a 100644
--- a/products/data_modeling/backend/tests/test_managed_viewset_providers.py
+++ b/products/data_modeling/backend/tests/test_managed_viewset_providers.py
@@ -27,7 +27,6 @@ from products.data_modeling.backend.facade.managed_viewset_hooks import (
)
from products.data_modeling.backend.facade.models import (
DAG,
- REVENUE_ANALYTICS_DAG_NAME,
DataWarehouseManagedViewSet,
DataWarehouseSavedQuery,
Node,
@@ -96,9 +95,9 @@ class TestManagedViewSetProviders(BaseTest):
self.assertFalse(view.is_materialized)
self.assertIsNone(view.sync_frequency_interval)
# A non-materialized view is computed at query time — it must never be scheduled for
- # materialization, nor get a managed (revenue-analytics) DAG.
+ # materialization.
mock_schedule.assert_not_called()
- self.assertFalse(DAG.objects.filter(team=self.team, name=REVENUE_ANALYTICS_DAG_NAME).exists())
+ self.assertEqual(DAG.objects.filter(team=self.team).count(), 1)
self.assertEqual(
list(Node.objects.filter(saved_query=view).values_list("dag", "type")),
[(DAG.get_or_create_default(self.team).id, NodeType.VIEW)],
diff --git a/products/data_warehouse/backend/models/test/test_managed_viewset.py b/products/data_warehouse/backend/models/test/test_managed_viewset.py
index 06b310675e50..392053a74e15 100644
--- a/products/data_warehouse/backend/models/test/test_managed_viewset.py
+++ b/products/data_warehouse/backend/models/test/test_managed_viewset.py
@@ -3,19 +3,9 @@ from unittest.mock import patch
from django.db import IntegrityError
-from parameterized import parameterized
-
from posthog.schema import RevenueAnalyticsEventItem, RevenueCurrencyPropertyConfig
-from products.data_modeling.backend.facade.models import (
- DAG,
- REVENUE_ANALYTICS_DAG_NAME,
- DataModelingJob,
- DataWarehouseManagedViewSet,
- DataWarehouseSavedQuery,
- Node,
- NodeType,
-)
+from products.data_modeling.backend.facade.models import DAG, DataWarehouseManagedViewSet, DataWarehouseSavedQuery, Node
from products.warehouse_sources.backend.facade.models import (
DataWarehouseCredential,
DataWarehouseTable,
@@ -201,56 +191,7 @@ class TestDataWarehouseManagedViewSetModel(BaseTest):
list(Node.objects.filter(team=self.team, saved_query=view).values_list("dag", flat=True)),
[default_dag.id],
)
- self.assertFalse(DAG.objects.filter(team=self.team, name=REVENUE_ANALYTICS_DAG_NAME).exists())
-
- @parameterized.expand([("only_in_managed_dag", False), ("also_in_default_dag", True)])
- def test_sync_views_moves_node_out_of_revenue_analytics_dag(self, _name, also_in_default_dag):
- managed_viewset = DataWarehouseManagedViewSet.objects.create(
- team=self.team,
- kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS,
- )
- managed_viewset.sync_views()
- sq = (
- DataWarehouseSavedQuery.objects.filter(team=self.team, managed_viewset=managed_viewset)
- .exclude(deleted=True)
- .first()
- )
- default_dag = DAG.get_or_create_default(self.team)
- ra_dag = DAG.get_or_create_revenue_analytics(self.team)
- Node.objects.filter(saved_query=sq).delete()
- managed_node = Node.objects.create(team=self.team, saved_query=sq, dag=ra_dag, type=NodeType.VIEW)
- if also_in_default_dag:
- Node.objects.create(team=self.team, saved_query=sq, dag=default_dag, type=NodeType.VIEW)
-
- managed_viewset.sync_views()
-
- nodes = list(Node.objects.filter(team=self.team, saved_query=sq))
- self.assertEqual([node.dag_id for node in nodes], [default_dag.id])
- self.assertEqual(nodes[0].id == managed_node.id, not also_in_default_dag)
- self.assertFalse(Node.objects.filter(dag=ra_dag, saved_query__isnull=False).exists())
-
- def test_sync_views_leaves_a_materializing_node_in_the_revenue_analytics_dag(self):
- managed_viewset = DataWarehouseManagedViewSet.objects.create(
- team=self.team,
- kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS,
- )
- managed_viewset.sync_views()
- sq = (
- DataWarehouseSavedQuery.objects.filter(team=self.team, managed_viewset=managed_viewset)
- .exclude(deleted=True)
- .first()
- )
- ra_dag = DAG.get_or_create_revenue_analytics(self.team)
- Node.objects.filter(saved_query=sq).delete()
- managed_node = Node.objects.create(team=self.team, saved_query=sq, dag=ra_dag, type=NodeType.MAT_VIEW)
- DataModelingJob.objects.create(team=self.team, saved_query=sq, status=DataModelingJob.Status.RUNNING)
-
- managed_viewset.sync_views()
-
- self.assertEqual(
- list(Node.objects.filter(team=self.team, saved_query=sq).values_list("id", "dag_id")),
- [(managed_node.id, ra_dag.id)],
- )
+ self.assertEqual(list(DAG.objects.filter(team=self.team).values_list("id", flat=True)), [default_dag.id])
def test_delete_with_views(self):
"""Test that delete_with_views properly deletes the managed viewset and marks views as deleted"""
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/editing.py b/products/data_warehouse/backend/presentation/views/saved_query/editing.py
index 6597a80ebaa9..aec2cce5a383 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/editing.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/editing.py
@@ -41,7 +41,6 @@ from products.data_modeling.backend.facade.modeling import ResolutionCycleError,
from products.data_modeling.backend.facade.models import (
DataWarehouseSavedQuery,
DataWarehouseSavedQueryColumnAnnotation,
- Node,
validate_saved_query_name,
)
from products.data_tools.backend.facade.models import DataWarehouseSavedQueryFolder
@@ -578,23 +577,16 @@ class DataWarehouseSavedQuerySerializer(
detail=Detail(name=view.name, changes=changes),
)
- # Best effort sync to the data modeling DAG representation. It reads the node's
- # placement, so it stays under the row lock a move also takes: outside it a move can
- # land between the read and the sync, which would then create a second node in the
- # DAG the move just left. The savepoint keeps a failed sync best effort without
- # poisoning the update's own transaction.
+ # Best effort sync to the data modeling DAG representation. The savepoint keeps a
+ # failed sync best effort without poisoning the update's own transaction.
if "query" in validated_data:
- node = Node.objects.filter(team_id=view.team_id, saved_query=view).select_related("dag").first()
- dag = node.dag if node else None
try:
with transaction.atomic():
- modeling_api.sync_saved_query_to_dag(view, dag=dag)
+ modeling_api.sync_saved_query_to_dag(view)
except Exception as e:
capture_exception(e)
logger.exception("Failed to sync saved query to DAG", saved_query_name=view.name)
- modeling_api.adopt_saved_query_without_edges(
- view.team_id, view.id, _lineage_sync_error_detail(e), dag_id=dag.id if dag else None
- )
+ modeling_api.adopt_saved_query_without_edges(view.team_id, view.id, _lineage_sync_error_detail(e))
self._report_view_action(
"view updated",
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/lineage.py b/products/data_warehouse/backend/presentation/views/saved_query/lineage.py
index 51d4cb70890e..ab1d63069dd4 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/lineage.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/lineage.py
@@ -51,16 +51,11 @@ def _parse_level(request: request.Request) -> int | None:
def _related_saved_queries(saved_query: DataWarehouseSavedQuery, *, upstream: bool, max_depth: int | None) -> set[str]:
"""Walk the data modeling graph from a saved query and identify what it reaches.
- The walk starts from every node that stands for the query and unions what they reach: a
- saved query can hold a node in more than one DAG, and a query in a managed DAG is also
- represented by a proxy table node in each DAG that reads from it. Without the proxies a
- consumer would report the managed query upstream while the managed query reported no
- consumer downstream.
+ The walk starts from every node that stands for the query and unions what they reach.
Each reached node is identified the way the lineage endpoints have always identified it: a
- query by its saved-query id, an imported warehouse table by its `DataWarehouseTable` id, a
- cross-DAG proxy by the id of the saved query it stands in for, and a PostHog system table by
- its name. A table node carries the id in its properties
+ query by its saved-query id, an imported warehouse table by its `DataWarehouseTable` id, and a
+ PostHog system table by its name. A table node carries the id in its properties
(`saved_query_dag_sync.resolve_dependency_to_node`) rather than the `saved_query` FK.
"""
start_node_ids = saved_query_node_ids(saved_query.team_id, saved_query.id)
@@ -78,8 +73,6 @@ def _related_saved_queries(saved_query: DataWarehouseSavedQuery, *, upstream: bo
continue
if saved_query_id is not None:
identifiers.add(str(saved_query_id))
- elif properties.get("saved_query_id"):
- identifiers.add(str(properties["saved_query_id"]))
elif properties.get("warehouse_table_id"):
identifiers.add(str(properties["warehouse_table_id"]))
else:
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/sync_cadence.py b/products/data_warehouse/backend/presentation/views/saved_query/sync_cadence.py
index ed46a092af82..1646031805d3 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/sync_cadence.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/sync_cadence.py
@@ -170,8 +170,6 @@ def visible_blocker_names(
return {}
visible: dict[str, str] = {}
- # Lists, not single ids: one saved query or table can hold a node in several DAGs, and a grant
- # covers the resource, so every node of it becomes visible together.
node_ids_by_saved_query: dict[str, list[str]] = {}
node_ids_by_table: dict[str, list[str]] = {}
for node_id in node_ids:
diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.py
index 8c0dd98199ee..6ec00c02dd4b 100644
--- a/products/data_warehouse/backend/tests/api/test_saved_query.py
+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py
@@ -1692,10 +1692,7 @@ class TestSavedQuery(APIBaseTest):
child_ancestors = response.json()["descendants"]
self.assertEqual(child_ancestors, [])
- def test_ancestors_identify_warehouse_and_cross_dag_nodes_by_id(self):
- # An imported warehouse table and a cross-DAG proxy are lineage nodes with no saved_query
- # FK; the id lives in properties. Both must be returned by id, not by name, so the answer
- # stays resolvable for a caller. These node shapes mirror resolve_dependency_to_node.
+ def test_ancestors_identify_warehouse_table_nodes_by_id(self):
dag = DAG.get_or_create_default(self.team)
consumer = DataWarehouseSavedQuery.objects.create(
team=self.team,
@@ -1704,7 +1701,6 @@ class TestSavedQuery(APIBaseTest):
created_by=self.user,
)
consumer_node = Node.objects.create(team=self.team, saved_query=consumer, dag=dag, type=NodeType.VIEW)
-
warehouse_table = DataWarehouseTable.objects.create(
team=self.team, name="stripe_charges", format="Parquet", url_pattern="s3://bucket/path"
)
@@ -1715,65 +1711,14 @@ class TestSavedQuery(APIBaseTest):
type=NodeType.TABLE,
properties={"origin": "warehouse", "warehouse_table_id": str(warehouse_table.id)},
)
-
- revenue_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="revenue_view",
- query={"kind": "HogQLQuery", "query": "select 1"},
- created_by=self.user,
- )
- proxy_node = Node.objects.create(
- team=self.team,
- dag=dag,
- name="revenue_view",
- type=NodeType.TABLE,
- properties={"origin": "cross_dag_view", "saved_query_id": str(revenue_query.id)},
- )
-
Edge.objects.create(team=self.team, dag=dag, source=warehouse_node, target=consumer_node)
- Edge.objects.create(team=self.team, dag=dag, source=proxy_node, target=consumer_node)
-
- # The query the proxy stands in for has its real node in the managed DAG, with its own
- # consumer there. Its descendants must union both: the managed consumer through the real
- # node and the Default-DAG consumer through the proxy, or the relationship is one-way.
- managed_dag = DAG.objects.create(team=self.team, name="PostHog Revenue Analytics")
- revenue_node = Node.objects.create(
- team=self.team, saved_query=revenue_query, dag=managed_dag, type=NodeType.VIEW
- )
- managed_consumer = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="managed_consumer_view",
- query={"kind": "HogQLQuery", "query": "select 1"},
- created_by=self.user,
- )
- managed_consumer_node = Node.objects.create(
- team=self.team, saved_query=managed_consumer, dag=managed_dag, type=NodeType.VIEW
- )
- Edge.objects.create(team=self.team, dag=managed_dag, source=revenue_node, target=managed_consumer_node)
response = self.client.post(
f"/api/environments/{self.team.id}/warehouse_saved_queries/{consumer.id}/ancestors",
)
- self.assertEqual(response.status_code, 200, response.content)
- self.assertEqual(
- sorted(response.json()["ancestors"]),
- sorted([str(warehouse_table.id), str(revenue_query.id)]),
- )
-
- response = self.client.post(
- f"/api/environments/{self.team.id}/warehouse_saved_queries/{revenue_query.id}/descendants",
- )
- self.assertEqual(response.status_code, 200, response.content)
- self.assertEqual(
- sorted(response.json()["descendants"]),
- sorted([str(consumer.id), str(managed_consumer.id)]),
- )
- response = self.client.get(
- f"/api/environments/{self.team.id}/warehouse_saved_queries/{revenue_query.id}/dependencies",
- )
self.assertEqual(response.status_code, 200, response.content)
- self.assertEqual(response.json(), {"upstream_count": 0, "downstream_count": 2})
+ self.assertEqual(response.json()["ancestors"], [str(warehouse_table.id)])
def test_descendants_omit_a_metric_that_reads_the_view(self):
dag = DAG.get_or_create_default(self.team)
diff --git a/products/model_crossing_uses_baseline.txt b/products/model_crossing_uses_baseline.txt
index 0c6f241cde9b..3a77c4376666 100644
--- a/products/model_crossing_uses_baseline.txt
+++ b/products/model_crossing_uses_baseline.txt
@@ -132,7 +132,6 @@ data_catalog.TableCertification products.data_catalog.backend.facade.api.depreca
data_catalog.TableCertification products.data_catalog.backend.facade.api.deprecate facade-returns 1
data_catalog.TableCertification products.data_catalog.backend.facade.api.propose_certification facade-returns 1
data_catalog.TableCertification products.data_catalog.backend.facade.api.revoke_certification facade-accepts(cert) 1
-data_modeling.DAG products.data_modeling.backend.facade.api.sync_saved_query_to_dag facade-accepts(dag) 1
data_modeling.DataModelingJob products.data_modeling.backend.facade.api.latest_saved_query_materialization_job facade-returns 1
data_modeling.DataWarehouseSavedQuery products.customer_analytics.backend.facade.api get_model 5
data_modeling.DataWarehouseSavedQuery products.data_modeling.backend.facade.api.apply_saved_query_frequency_anchor facade-accepts(saved_query) 1