Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-modeling): retire DataWarehouseModelPath (#118747)

Thiago Salvatore65d5a45b8638
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yaml
index 84d4526bf5fa..ca41629ef73d 100644
--- a/.semgrep/rules/security/idor-team-scoped-models.yaml
+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml
@@ -159,7 +159,6 @@ rules:
|DataWarehouseExpression
|DataWarehouseJoin
|DataWarehouseManagedViewSet
- |DataWarehouseModelPath
|DataWarehouseSavedQuery
|DataWarehouseSavedQueryColumnAnnotation
|DataWarehouseSavedQueryDraft
@@ -631,7 +630,6 @@ rules:
|DataWarehouseExpression
|DataWarehouseJoin
|DataWarehouseManagedViewSet
- |DataWarehouseModelPath
|DataWarehouseSavedQuery
|DataWarehouseSavedQueryColumnAnnotation
|DataWarehouseSavedQueryDraft
@@ -1042,7 +1040,6 @@ rules:
|DataWarehouseCredential
|DataWarehouseJoin
|DataWarehouseManagedViewSet
- |DataWarehouseModelPath
|DataWarehouseSavedQuery
|DataWarehouseSavedQueryDraft
|DataWarehouseSavedQueryFolder
diff --git a/docs/internal/hogql-trino-compiler.md b/docs/internal/hogql-trino-compiler.md
index a218952758bf..1d2604e80ea2 100644
--- a/docs/internal/hogql-trino-compiler.md
+++ b/docs/internal/hogql-trino-compiler.md
@@ -250,7 +250,7 @@ Compilation uses current source bindings, saved-query dependencies, filters, and
Saved translation results remain diagnostic records and are not prerequisites or execution inputs for shadow jobs.
Compilation failures fail the modeling job, without connecting to Trino or falling back to DuckDB.
-Trino replaces the output table in the organization's catalog under `posthog_data_modeling_team_<team_id>`, using the sanitized model-path label or saved-query UUID.
+Trino replaces the output table in the organization's catalog under `posthog_data_modeling_team_<team_id>`, using the sanitized saved-query UUID.
The compiler and materializer share this naming policy, so each compiled query can read upstream model outputs directly.
The legacy Duckgres path retains `shadow_<team_id>_models` and normalized saved-query names.
The shadow is a separate DAG run, so ClickHouse materialization and publication never wait for Trino.
diff --git a/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt b/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt
index 1f33eb358afa..e099d8127c54 100644
--- a/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt
+++ b/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt
@@ -55,3 +55,4 @@ posthog.1383_taggeditem_untrack_legacy_keys
ee.0064_split_identity_provider_config_scopes
conversations.0076_tag_legacy_conversation_attachments
posthog.1398_remove_userproductlist_state
+data_modeling.0047_remove_datawarehousemodelpath_state
diff --git a/posthog/models/scoping/baseline_unmigrated.txt b/posthog/models/scoping/baseline_unmigrated.txt
index fc12bcbf3572..01b337369a3e 100644
--- a/posthog/models/scoping/baseline_unmigrated.txt
+++ b/posthog/models/scoping/baseline_unmigrated.txt
@@ -42,7 +42,6 @@ DataModelingJob
DataWarehouseCredential
DataWarehouseJoin
DataWarehouseManagedViewSet
-DataWarehouseModelPath
DataWarehouseSavedQuery
DataWarehouseSavedQueryDraft
DataWarehouseSavedQueryFolder
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index c2f446532ff1..25d3fbec05e3 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -109,7 +109,6 @@
1 products/data_modeling/backend/logic/schedule_reconcile.py
1 products/data_modeling/backend/logic/tier_membership.py
1 products/data_modeling/backend/models/datawarehouse_managed_viewset.py
-1 products/data_modeling/backend/models/modeling.py
1 products/data_warehouse/backend/logic/data_load/create_table.py
1 products/data_warehouse/backend/logic/webhook_consumer/config.py
1 products/demo/backend/logic/products/hedgebox/matrix.py
diff --git a/products/data_modeling/backend/facade/modeling.py b/products/data_modeling/backend/facade/modeling.py
index f45b78fb92fd..ce9033ae5ea2 100644
--- a/products/data_modeling/backend/facade/modeling.py
+++ b/products/data_modeling/backend/facade/modeling.py
@@ -1,21 +1,10 @@
-"""
-Model-path / resolver wiring for data_modeling.
-
-Re-exports the saved-query DAG resolver surface from ``models.modeling`` — the
-``DataWarehouseModelPath`` model, the bounded resolver, and its ``NodeType`` (note: a
-distinct enum from ``models.node.NodeType``, which ``facade.models`` exposes). Light at
-module level (modeling's heavy deps are deferred inside its methods).
-"""
-
from products.data_modeling.backend.models.modeling import (
DEFAULT_RESOLUTION_DEADLINE_SECONDS,
DEFAULT_RESOLUTION_MAX_VIEW_DEPTH,
RESOLUTION_SOURCE_LINEAGE,
RESOLUTION_SOURCE_MATERIALIZATION,
BoundedResolver,
- DataWarehouseModelPath,
LabelTreeField,
- NodeType,
ResolutionCycleError,
ResolutionDepthExceededError,
ResolutionTimeoutError,
@@ -29,9 +18,7 @@ __all__ = [
"RESOLUTION_SOURCE_LINEAGE",
"RESOLUTION_SOURCE_MATERIALIZATION",
"BoundedResolver",
- "DataWarehouseModelPath",
"LabelTreeField",
- "NodeType",
"ResolutionCycleError",
"ResolutionDepthExceededError",
"ResolutionTimeoutError",
diff --git a/products/data_modeling/backend/facade/models.py b/products/data_modeling/backend/facade/models.py
index fd9ddde9082e..9d8a1faf00b2 100644
--- a/products/data_modeling/backend/facade/models.py
+++ b/products/data_modeling/backend/facade/models.py
@@ -6,9 +6,6 @@ their enums, and the in-module helpers cross-product object-consumers need (HogQ
builders that traverse the saved-query DAG, dispatch on ``isinstance``, or call model
methods). The models' heavy dependencies (HogQL, temporal) are deferred inside their
methods, so importing this adds nothing beyond the models Django loads at ``setup()``.
-
-Note: ``NodeType`` here is ``models.node.NodeType`` (table/view/matview/endpoint). The
-distinct resolver enum ``models.modeling.NodeType`` is exposed via ``facade.modeling``.
"""
from products.data_modeling.backend.graph import Graph
diff --git a/products/data_modeling/backend/logic/saved_query_materialization.py b/products/data_modeling/backend/logic/saved_query_materialization.py
index 1303e93d4f7e..21fc02bfcc43 100644
--- a/products/data_modeling/backend/logic/saved_query_materialization.py
+++ b/products/data_modeling/backend/logic/saved_query_materialization.py
@@ -83,7 +83,6 @@ def _enable_materialization(
previously_materialized = saved_query.is_materialized
_save_materialization(saved_query, is_materialized=True, sync_frequency_interval=sync_frequency_interval)
- # Enable materialization - this handles model path setup and schedule creation
# If this fails, it will set is_materialized = False
try:
saved_query.schedule_materialization(trigger_immediate_run=True, triggered_by_id=user.pk)
diff --git a/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py b/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py
index 28833a2d3a0c..16772567343b 100644
--- a/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py
+++ b/products/data_modeling/backend/management/commands/cleanup_orphaned_matview_tables.py
@@ -2,11 +2,10 @@
The proper delete path (`delete_saved_query`) does four things: remove the DAG node
(`delete_node_from_dag`), soft-delete the query's joins, `revert_materialization()` (soft-delete the
-backing table + null table_id + drop model paths + clear schedule/tier), and `soft_delete()` (rename
+backing table + null table_id + clear schedule/tier), and `soft_delete()` (rename
to POSTHOG_DELETED). A query soft-deleted another way — a manual `deleted=True` in a shell, bypassing
the model method — sets only that flag, leaving the table live (leaks into the "self-managed sources"
-sidebar), the DAG node in place (a ghost node with no resolvable saved query), and the model paths /
-joins dangling.
+sidebar), the DAG node in place (a ghost node with no resolvable saved query), and the joins dangling.
This finds those half-deleted matviews and runs the rest of the cascade, so each ends in the same
state a proper delete would have produced. Dry-run by default; --apply to mutate; --team-id to scope.
@@ -138,7 +137,7 @@ def cascade_delete(saved_query: DataWarehouseSavedQuery) -> str:
).exclude(deleted=True):
join.soft_delete()
- saved_query.revert_materialization() # soft-deletes the table, nulls table_id, drops model paths
+ saved_query.revert_materialization()
if saved_query.deleted_name is None: # complete the rename a bypassed delete skipped
saved_query.soft_delete()
return CLEANED
diff --git a/products/data_modeling/backend/migrations/0047_remove_datawarehousemodelpath_state.py b/products/data_modeling/backend/migrations/0047_remove_datawarehousemodelpath_state.py
new file mode 100644
index 000000000000..ef18a18f793f
--- /dev/null
+++ b/products/data_modeling/backend/migrations/0047_remove_datawarehousemodelpath_state.py
@@ -0,0 +1,23 @@
+from django.db import migrations
+
+from posthog.migration_helpers import DropForeignKey
+
+
+class Migration(migrations.Migration):
+ atomic = False
+
+ dependencies = [
+ ("data_modeling", "0046_dwsavedquery_team_live_created_idx"),
+ ]
+
+ operations = [
+ migrations.SeparateDatabaseAndState(
+ state_operations=[migrations.DeleteModel(name="DataWarehouseModelPath")],
+ database_operations=[
+ DropForeignKey("posthog_datawarehousemodelpath", column="team_id"),
+ DropForeignKey("posthog_datawarehousemodelpath", column="created_by_id"),
+ DropForeignKey("posthog_datawarehousemodelpath", column="saved_query_id"),
+ DropForeignKey("posthog_datawarehousemodelpath", column="table_id"),
+ ],
+ ),
+ ]
diff --git a/products/data_modeling/backend/migrations/max_migration.txt b/products/data_modeling/backend/migrations/max_migration.txt
index 0eddef7ef760..56322fc8adf9 100644
--- a/products/data_modeling/backend/migrations/max_migration.txt
+++ b/products/data_modeling/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0046_dwsavedquery_team_live_created_idx
+0047_remove_datawarehousemodelpath_state
diff --git a/products/data_modeling/backend/models/__init__.py b/products/data_modeling/backend/models/__init__.py
index 2d4e8fb1c9e1..c48017eab4b2 100644
--- a/products/data_modeling/backend/models/__init__.py
+++ b/products/data_modeling/backend/models/__init__.py
@@ -8,7 +8,6 @@ from .edge import CycleDetectionError, DAGMismatchError, DataModelingEdgeManager
from .github_sync_config import GitHubSyncConfig, GitHubSyncStatus
from .github_sync_plan import GitHubSyncPlan, GitHubSyncPlanStatus
from .github_synced_model import GitHubSyncedModel
-from .modeling import DataWarehouseModelPath
from .node import Node, NodeType
__all__ = [
@@ -24,7 +23,6 @@ __all__ = [
"DataModelingJobEngine",
"DataModelingJobStatus",
"DataWarehouseManagedViewSet",
- "DataWarehouseModelPath",
"DataWarehouseSavedQuery",
"DataWarehouseSavedQueryColumnAnnotation",
"DataWarehouseSavedQueryDraft",
diff --git a/products/data_modeling/backend/models/datawarehouse_managed_viewset.py b/products/data_modeling/backend/models/datawarehouse_managed_viewset.py
index e4bb8cda1ed7..863c3bedea4a 100644
--- a/products/data_modeling/backend/models/datawarehouse_managed_viewset.py
+++ b/products/data_modeling/backend/models/datawarehouse_managed_viewset.py
@@ -140,7 +140,7 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
orphaned_views_to_revert: list[DataWarehouseSavedQuery] = []
# Keep this transaction short: persist DB changes only. The post-commit work
- # (schedule_materialization → Temporal RPCs + DataWarehouseModelPath updates,
+ # (schedule_materialization → Temporal RPCs,
# and orphan revert_materialization) runs outside the atomic block so row
# locks on posthog_datawarehousesavedquery aren't held across synchronous
# Temporal RPCs.
diff --git a/products/data_modeling/backend/models/datawarehouse_saved_query.py b/products/data_modeling/backend/models/datawarehouse_saved_query.py
index 545fde810bf4..c786b5f72701 100644
--- a/products/data_modeling/backend/models/datawarehouse_saved_query.py
+++ b/products/data_modeling/backend/models/datawarehouse_saved_query.py
@@ -293,14 +293,6 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
def name_chain(self) -> list[str]:
return self.name.split(".")
- def setup_model_paths(self):
- from products.data_modeling.backend.models.modeling import DataWarehouseModelPath
-
- if not DataWarehouseModelPath.objects.filter(team=self.team, saved_query=self).exists():
- DataWarehouseModelPath.objects.create_from_saved_query(self)
- else:
- DataWarehouseModelPath.objects.update_from_saved_query(self)
-
def schedule_materialization(
self, reconcile: bool = True, trigger_immediate_run: bool = False, triggered_by_id: int | None = None
):
@@ -430,7 +422,6 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
def revert_materialization(self):
from products.data_modeling.backend.logic.node_suspension import unsuspend_saved_query
from products.data_modeling.backend.logic.schedule_reconcile import apply_saved_query_frequency_target
- from products.data_modeling.backend.models.modeling import DataWarehouseModelPath
self.sync_frequency_interval = None
self.last_run_at = None
@@ -445,7 +436,6 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
self.table_id = None
self.save()
- DataWarehouseModelPath.objects.filter(team=self.team, path__lquery=f"*{{1,}}.{self.id.hex}").delete()
# A reverted matview must also leave its cadence tier, or it would keep being
# materialized on tiered v2. Best-effort — the revert itself already succeeded.
diff --git a/products/data_modeling/backend/models/modeling.py b/products/data_modeling/backend/models/modeling.py
index 01ba8c4d11af..196a0d004fa2 100644
--- a/products/data_modeling/backend/models/modeling.py
+++ b/products/data_modeling/backend/models/modeling.py
@@ -1,12 +1,7 @@
-import enum
import time
-import uuid
-import dataclasses
from typing import ClassVar
-from django.contrib.postgres import indexes as pg_indexes
-from django.core.exceptions import ObjectDoesNotExist
-from django.db import connection, models, transaction
+from django.db import models
from prometheus_client import Counter, Histogram
@@ -15,19 +10,14 @@ from posthog.hogql.constants import HogQLDialect
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
from posthog.hogql.database.models import SavedQuery
-from posthog.hogql.database.s3_table import DataWarehouseTable as HogQLDataWarehouseTable
from posthog.hogql.errors import QueryError
from posthog.hogql.parser import parse_select
from posthog.hogql.resolver import Resolver, ResolverFactory
from posthog.hogql.visitor import TraversingVisitor
from posthog.models.team import Team
-from posthog.models.user import User
-from posthog.models.utils import CreatedMetaFields, UpdatedMetaFields, UUIDTModel
from products.data_modeling.backend.facade.system_tables import DATA_MODELING_ALLOWED_SYSTEM_TABLES
-from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-from products.warehouse_sources.backend.facade.models import DataWarehouseTable
LabelPath = list[str]
@@ -395,42 +385,6 @@ class LabelTreeField(models.Field):
return ".".join(value)
-class LabelQuery(models.Lookup):
- """Implement a lookup for an ltree label query using the ~ operator."""
-
- lookup_name = "lquery"
-
- def __init__(self, *args, **kwargs):
- self.prepare_rhs = False
- super().__init__(*args, **kwargs)
-
- def as_sql(self, compiler, connection):
- lhs, lhs_params = self.process_lhs(compiler, connection)
- rhs, rhs_params = self.process_rhs(compiler, connection)
- params = [*lhs_params, *rhs_params]
- return "%s ~ %s" % (lhs, rhs), params # noqa: UP031
-
-
-class LabelQueryArray(models.Lookup):
- """Implement a lookup for an array of ltree label queries using the ? operator."""
-
- lookup_name = "lqueryarray"
-
- def __init__(self, *args, **kwargs):
- self.prepare_rhs = False
- super().__init__(*args, **kwargs)
-
- def as_sql(self, compiler, connection):
- lhs, lhs_params = self.process_lhs(compiler, connection)
- rhs, rhs_params = self.process_rhs(compiler, connection)
- params = [*lhs_params, *rhs_params]
- return "%s ? %s" % (lhs, rhs), params # noqa: UP031
-
-
-LabelTreeField.register_lookup(LabelQuery)
-LabelTreeField.register_lookup(LabelQueryArray)
-
-
# enclosing WITH clauses, innermost first — a name resolves against the closest one
CteScope = tuple[dict[str, ast.CTE], ...]
@@ -659,98 +613,6 @@ def get_parents_from_model_query(
return parents
-class NodeType(enum.Enum):
- SAVED_QUERY = "SavedQuery"
- POSTHOG = "PostHog"
- TABLE = "Table"
-
-
-NodeId = str
-Node = tuple[NodeId, NodeType]
-Edge = tuple[NodeId, NodeId]
-
-
-class DAG:
- """A basic DAG composed of nodes and edges."""
-
- edges: set[Edge]
- nodes: set[Node]
-
-
-UPDATE_PATHS_QUERY = """\
-insert into posthog_datawarehousemodelpath (
- id,
- team_id,
- table_id,
- saved_query_id,
- path,
- created_by_id,
- created_at,
- updated_at
-) (
- select distinct on (id)
- id,
- team_id,
- table_id,
- saved_query_id,
- parent.path || subpath(model_path.path, index(model_path.path, text2ltree(%(child)s))) as path,
- created_by_id,
- created_at,
- now() as updated_at
- from
- posthog_datawarehousemodelpath as model_path,
- (
- select
- path
- from posthog_datawarehousemodelpath
- where path ~ ('*.' || %(parent)s)::lquery
- and team_id = %(team_id)s
- ) as parent
- where
- model_path.path ~ ('*.' || %(child)s || '.*')::lquery
- and team_id = %(team_id)s
- order by id, path
-)
-on conflict (id) do
-update
- set path = EXCLUDED.path,
- updated_at = now()
-"""
-
-DELETE_DUPLICATE_PATHS_QUERY = """\
-delete from posthog_datawarehousemodelpath
-where
- team_id = %(team_id)s
- and id in (
- select id
- from (
- select id, row_number() over (partition by team_id, path) as row_number
- from posthog_datawarehousemodelpath
- ) partitioned
- where partitioned.row_number > 1
-);
-"""
-
-CYCLE_CHECK_QUERY = """\
-select exists (
- select 1
- from posthog_datawarehousemodelpath
- where team_id = %(team_id)s
- and path ~ ('*.' || %(child)s || '.*.' || %(parent)s)::lquery
-)
-"""
-
-
-class ModelPathCycleError(Exception):
- """Exception raised when a cycle would be created in the model DAG."""
-
- def __init__(self, child: str, parent: str):
- super().__init__(f"Adding {parent} as a parent of {child} would create a cycle in the DAG")
- self.child = child
- self.parent = parent
-
-
class UnknownParentError(Exception):
"""Exception raised when the parent for a model is not found."""
@@ -759,356 +621,3 @@ class UnknownParentError(Exception):
f"The parent name {parent} does not correspond to an existing PostHog table, Data Warehouse Table, or Data Warehouse Saved Query."
)
self.query = query
-
-
-class ModelPathAlreadyExistsError(Exception):
- """Exception raised when trying to create paths for a model that already has some."""
-
- def __init__(self, model_name: str):
- super().__init__(f"Model {model_name} cannot be created as it already exists")
-
-
-class ModelPathDoesNotExistError(Exception):
- """Exception raised when trying to update paths for a model that doesn't exist."""
-
- def __init__(self, model_name: str):
- super().__init__(f"Model {model_name} doesn't exist")
-
-
-class DataWarehouseModelPathManager(models.Manager["DataWarehouseModelPath"]):
- """A model manager that implements some common path operations."""
-
- def create_from_saved_query(self, saved_query: DataWarehouseSavedQuery) -> "list[DataWarehouseModelPath]":
- """Create one or more model paths from a new `DataWarehouseSavedQuery`.
-
- Creating one or more model paths from a new `DataWarehouseSavedQuery` is straight-forward as we
- don't have to worry about this model having its own children paths that need updating: We are
- only adding a leaf node to all parents' paths. We check this model indeed does not exist to
- ensure that is the case.
-
- Raises:
- ValueError: If no paths exists for the provided `DataWarehouseSavedQuery`.
- """
- model_query_value = saved_query.query.get("query") if isinstance(saved_query.query, dict) else None
- model_query = model_query_value if isinstance(model_query_value, str) else ""
-
- return self.create_leaf_paths_from_query(
- team=saved_query.team,
- model_name=saved_query.name,
- model_query=model_query,
- saved_query_id=saved_query.id,
- created_by=saved_query.created_by,
- label=saved_query.id.hex,
- )
-
- def create_leaf_paths_from_query(
- self,
- team: Team,
- model_name: str,
- model_query: str,
- label: str,
- saved_query_id: uuid.UUID,
- created_by: User | None = None,
- table_id: uuid.UUID | None = None,
- ) -> "list[DataWarehouseModelPath]":
- """Create all paths to a new leaf model.
-
- A path will be created for each parent, as extracted from the given query.
- """
- with transaction.atomic():
- if self.filter(team=team, saved_query_id=saved_query_id).exists():
- raise ModelPathAlreadyExistsError(saved_query_id.hex)
-
- parent_paths = self.get_or_create_query_parent_paths(team, model_name, model_query)
-
- # If we don't have any parent paths then we can treat ourselves as a root node
- # This can happen when creating a query that returns a static set of rows, like a SELECT 1.e
- paths = [[*model_path.path, label] for model_path in parent_paths]
- if not paths:
- paths = [[label]]
-
- results: list[DataWarehouseModelPath] = []
- for path in paths:
- model, _ = DataWarehouseModelPath.objects.get_or_create(
- team=team,
- saved_query_id=saved_query_id,
- table_id=table_id,
- path=path,
- defaults={"created_by": created_by},
- )
- results.append(model)
-
- return results
-
- def get_or_create_root_path_for_posthog_source(
- self, posthog_source_name: str, team: Team
- ) -> tuple["DataWarehouseModelPath", bool]:
- """Get a root path for a PostHog source, creating it if it doesn't exist.
-
- PostHog sources are well-known PostHog tables. We check against the team's HogQL database
- to ensure that the source exists before creating the path.
-
- Raises:
- ValueError: If the provided `posthog_source_name` is not a PostHog table.
-
- Returns:
- A tuple with the model path and a `bool` indicating whether it was created or not.
- """
- try:
- self.get_hogql_database(team).get_table(posthog_source_name)
- except QueryError:
- raise ValueError(f"Provided source {posthog_source_name} is not a PostHog table")
-
- return self.get_or_create(path=[posthog_source_name], team=team, defaults={"saved_query": None})
-
- def get_hogql_database(self, team: Team) -> Database:
- """Get the HogQL database for given team."""
- # Internal model-path resolution (no user); bypass warehouse HogQL access control.
- return Database.create_for(
- team=team,
- bypass_warehouse_access_control=True,
- allowed_system_tables=DATA_MODELING_ALLOWED_SYSTEM_TABLES,
- )
-
- def get_or_create_root_path_for_data_warehouse_table(
- self, data_warehouse_table: DataWarehouseTable
- ) -> tuple["DataWarehouseModelPath", bool]:
- """Get a root path for a `DataWarehouseTable`, creating it if it doesn't exist.
-
- A `DataWarehouseTable` is loaded by us into S3 or read directly from an external data source,
- like our user's S3 bucket or their PostgreSQL database.
-
- Either way, it is a table we can consider a root node, as it's managed by data warehouse
- data import workflows.
-
- Returns:
- A tuple with the model path and a `bool` indicating whether it was created or not.
- """
- table_id = data_warehouse_table.id
- return self.get_or_create(
- path=[table_id.hex],
- team=data_warehouse_table.team,
- defaults={"saved_query": None, "table": data_warehouse_table},
- )
-
- def filter_all_leaf_paths(self, leaf_id: str | uuid.UUID, team: Team):
- """Filter all paths to leaf node given by `leaf_id`."""
- if isinstance(leaf_id, uuid.UUID):
- leaf_id = leaf_id.hex
- return self.filter(team=team, path__lquery=f"*.{leaf_id}")
-
- def get_or_create_query_parent_paths(
- self, team: Team, model_name: str, model_query: str
- ) -> list["DataWarehouseModelPath"]:
- """Get a list of model paths for a query's parents, creating root nodes if they do not exist."""
- parent_paths = []
- for parent in get_parents_from_model_query(team, model_name, model_query):
- try:
- parent_query = (
- DataWarehouseSavedQuery.objects.exclude(deleted=True).filter(team=team, name=parent).get()
- )
- except ObjectDoesNotExist:
- pass
- else:
- parent_query_paths = list(self.filter_all_leaf_paths(parent_query.id.hex, team=team).all())
- parent_paths.extend(parent_query_paths)
- continue
-
- try:
- table = self.get_hogql_database(team).get_table(parent)
- if not isinstance(table, HogQLDataWarehouseTable):
- raise ObjectDoesNotExist()
-
- if table.table_id:
- parent_table = (
- DataWarehouseTable.objects.exclude(deleted=True).filter(team=team, id=table.table_id).get()
- )
- else:
- parent_table = (
- DataWarehouseTable.objects.exclude(deleted=True).filter(team=team, name=table.name).get()
- )
-
- except (ObjectDoesNotExist, QueryError):
- pass
- else:
- parent_path, _ = self.get_or_create_root_path_for_data_warehouse_table(parent_table)
- parent_paths.append(parent_path)
- continue
-
- try:
- parent_path, _ = self.get_or_create_root_path_for_posthog_source(parent, team)
- except ValueError:
- pass
- else:
- parent_paths.append(parent_path)
- continue
-
- raise UnknownParentError(parent, model_query)
-
- return parent_paths
-
- def update_from_saved_query(self, saved_query: DataWarehouseSavedQuery) -> None:
- """Update model paths from an existing `DataWarehouseSavedQuery`."""
- if not self.filter(team=saved_query.team, saved_query=saved_query).exists():
- raise ValueError("Provided saved query contains no paths to update.")
-
- model_query_value = saved_query.query.get("query") if isinstance(saved_query.query, dict) else None
- model_query = model_query_value if isinstance(model_query_value, str) else ""
-
- self.update_paths_from_query(
- team=saved_query.team,
- model_name=saved_query.name,
- model_query=model_query,
- label=saved_query.id.hex,
- saved_query_id=saved_query.id,
- )
-
- def update_paths_from_query(
- self,
- team: Team,
- model_name: str,
- model_query: str,
- label: str,
- saved_query_id: uuid.UUID | None = None,
- table_id: uuid.UUID | None = None,
- ) -> None:
- """Update all model paths from a given query.
-
- We parse the query to extract all its direct parents. Then, we update all the paths
- that contain `label` to add an edge from parent and `label`, effectively removing the
- previous parent path.
-
- This may lead to duplicate paths, so we have to defer constraints, until the end of
- the transaction and clean them up.
- """
- parents = get_parents_from_model_query(team, model_name, model_query)
- # include_hidden surfaces `posthog.*`-namespaced tables (e.g. posthog.ai_events) so they
- # resolve as parents here too — matching the create path's get_table()-based resolution.
- database = self.get_hogql_database(team)
- posthog_table_names = set(database.get_posthog_table_names()) | set(
- database.get_posthog_table_names(include_hidden=True)
- )
-
- base_params = {
- "team_id": team.pk,
- "saved_query_id": saved_query_id,
- "table_id": table_id,
- }
-
- with transaction.atomic():
- with connection.cursor() as cursor:
- cursor.execute("SET CONSTRAINTS ALL DEFERRED")
-
- for parent in parents:
- if parent in posthog_table_names:
- parent_id = parent
- else:
- try:
- parent_query = (
- DataWarehouseSavedQuery.objects.exclude(deleted=True)
- .filter(team=team, name=parent)
- .get()
- )
- except ObjectDoesNotExist:
- try:
- table = self.get_hogql_database(team).get_table(parent)
- if not isinstance(table, HogQLDataWarehouseTable):
- raise ObjectDoesNotExist()
-
- if table.table_id:
- parent_table = (
- DataWarehouseTable.objects.exclude(deleted=True)
- .filter(team=team, id=table.table_id)
- .get()
- )
- else:
- parent_table = (
- DataWarehouseTable.objects.exclude(deleted=True)
- .filter(team=team, name=table.name)
- .get()
- )
- except (ObjectDoesNotExist, QueryError):
- raise UnknownParentError(parent, model_query)
- else:
- parent_id = parent_table.id.hex
- else:
- parent_id = parent_query.id.hex
-
- cursor.execute(CYCLE_CHECK_QUERY, params={"team_id": team.pk, "child": label, "parent": parent_id})
- cycle_exists_row = cursor.fetchone()
- if cycle_exists_row is not None and cycle_exists_row[0]:
- raise ModelPathCycleError(child=label, parent=parent_id)
-
- cursor.execute(UPDATE_PATHS_QUERY, params={**{"child": label, "parent": parent_id}, **base_params})
-
- cursor.execute(DELETE_DUPLICATE_PATHS_QUERY, params={"team_id": team.pk})
- cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
-
- def get_dag(self, team: Team):
- """Return a DAG of all the models for the given team.
-
- A DAG is composed by a set of edges and a set of nodes, where each node is a tuple
- of a node id and type, and each edge is a pair of two nodes. The edges are directed in the
- order of the tuple elements.
-
- TODO:
- * Should we resolve node id and node type to their underlying models?
- * Edges could be indexed by node if required by callers.
- * Certain paths can be redundant and could be excluded from the query.
- """
- edges = set()
- nodes: set[Node] = set()
- node_type: NodeType
- node_id: NodeId
-
- for model_path in self.filter(team=team).select_related("saved_query", "table").all():
- if model_path.saved_query is not None:
- node_type = NodeType.SAVED_QUERY
- elif model_path.table is not None:
- node_type = NodeType.TABLE
- else:
- node_type = NodeType.POSTHOG
-
- for index, node_id in enumerate(model_path.path):
- try:
- next_node_id = model_path.path[index + 1]
- except IndexError:
- node: tuple[NodeId, NodeType] = (node_id, node_type)
- nodes.add(node)
- else:
- edges.add((node_id, next_node_id))
-
- return DAG(edges=edges, nodes=nodes)
-
-
-class DataWarehouseModelPath(CreatedMetaFields, UpdatedMetaFields, UUIDTModel):
- """Django model to represent paths to a data warehouse model.
-
- A data warehouse model is represented by a saved query, and the path to it contains all
- tables and views that said query is selecting from, recursively all the way to root
- PostHog tables and external data source tables.
- """
-
- class Meta:
- indexes = [
- models.Index(fields=("team_id", "path"), name="team_id_path"),
- models.Index(fields=("team_id", "saved_query_id"), name="team_id_saved_query_id"),
- pg_indexes.GistIndex("path", name="model_path_path"),
- ]
- constraints = [
- models.UniqueConstraint(
- fields=("team_id", "path"), name="unique_team_id_path", deferrable=models.Deferrable.IMMEDIATE
- ),
- ]
- db_table = "posthog_datawarehousemodelpath"
-
- objects: DataWarehouseModelPathManager = DataWarehouseModelPathManager()
-
- path = LabelTreeField(null=False)
- team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+")
- table = models.ForeignKey(
- "warehouse_sources.DataWarehouseTable", null=True, default=None, on_delete=models.SET_NULL, related_name="+"
- )
- saved_query = models.ForeignKey(
- "data_modeling.DataWarehouseSavedQuery", null=True, default=None, on_delete=models.SET_NULL
- )
diff --git a/products/data_modeling/backend/tasks/cleanup_test_saved_queries.py b/products/data_modeling/backend/tasks/cleanup_test_saved_queries.py
index 885ea147fa31..d435e300e902 100644
--- a/products/data_modeling/backend/tasks/cleanup_test_saved_queries.py
+++ b/products/data_modeling/backend/tasks/cleanup_test_saved_queries.py
@@ -92,7 +92,6 @@ def cleanup_expired_test_saved_queries() -> None:
def _hard_delete_saved_query(saved_query: "DataWarehouseSavedQuery") -> None:
"""Hard-delete a single saved query and all downstream objects."""
from products.data_modeling.backend.logic.saved_query_dag_sync import delete_nodes
- from products.data_modeling.backend.models.modeling import DataWarehouseModelPath
from products.data_modeling.backend.models.node import Node
from products.data_tools.backend.models.join import DataWarehouseJoin
@@ -106,17 +105,15 @@ def _hard_delete_saved_query(saved_query: "DataWarehouseSavedQuery") -> None:
Q(team_id=saved_query.team_id)
& (Q(source_table_name=saved_query.name) | Q(joining_table_name=saved_query.name))
).delete()
- # 3. Delete model paths
- DataWarehouseModelPath.objects.filter(team=saved_query.team, path__lquery=f"*{{1,}}.{saved_query.id.hex}").delete()
- # 4. Revert materialization (drops schedule, soft-deletes the table)
+ # 3. Revert materialization (drops schedule, soft-deletes the table)
table_to_delete = saved_query.table
saved_query.revert_materialization()
- # 5. Delete S3 data for the materialized view
+ # 4. Delete S3 data for the materialized view
_delete_s3_data(saved_query)
- # 6. Hard-delete the materialized table if it exists
+ # 5. Hard-delete the materialized table if it exists
if table_to_delete is not None:
table_to_delete.delete()
- # 7. Hard-delete the saved query itself
+ # 6. Hard-delete the saved query itself
saved_query.delete()
diff --git a/products/data_modeling/backend/test/test_schedule_materialization.py b/products/data_modeling/backend/test/test_schedule_materialization.py
index 071154349341..68c4a8ed2505 100644
--- a/products/data_modeling/backend/test/test_schedule_materialization.py
+++ b/products/data_modeling/backend/test/test_schedule_materialization.py
@@ -50,13 +50,11 @@ class TestScheduleMaterializationV2Guard(BaseTest):
def test_skips_v1_and_nulls_frequency_when_dag_on_v2(self):
with (
mock.patch(GET_V2_DAG_IDS, return_value={str(self.dag.id)}),
- mock.patch.object(DataWarehouseSavedQuery, "setup_model_paths") as setup_paths,
mock.patch(f"{NODE_MAT}.sync_connect") as sync_connect,
mock.patch(f"{MODEL}.capture_exception") as capture,
self.captureOnCommitCallbacks(execute=True),
):
self.sq.schedule_materialization()
- setup_paths.assert_not_called()
# reporting on the healthy path would bury the one signal that matters
capture.assert_not_called()
# a frequency-only call carries no enable intent, so it must not start a one-off run
diff --git a/products/data_warehouse/backend/models/test/test_managed_viewset.py b/products/data_warehouse/backend/models/test/test_managed_viewset.py
index 56448ffced90..06b310675e50 100644
--- a/products/data_warehouse/backend/models/test/test_managed_viewset.py
+++ b/products/data_warehouse/backend/models/test/test_managed_viewset.py
@@ -449,13 +449,6 @@ class TestManagedViewSetSyncWithStripeSource(BaseTest):
self.assertQueryIsEmpty(customer_query)
def test_sync_views_schedules_after_transaction_commits(self):
- """schedule_materialization must run AFTER the phase 1 transaction commits,
- so that row locks on posthog_datawarehousesavedquery are released before
- synchronous Temporal RPCs and DataWarehouseModelPath updates begin. If
- schedule_materialization were called from inside the transaction, only the
- current iteration's saved query would be visible at the call site; after the
- refactor, all persisted saved queries should be visible at every call site.
- """
schemas = self._create_schemas_without_tables()
for schema in schemas:
self._create_table_for_schema(schema)
diff --git a/products/data_warehouse/backend/models/test/test_modeling.py b/products/data_warehouse/backend/models/test/test_modeling.py
index a439c989c1e8..e59fbaf5c2bd 100644
--- a/products/data_warehouse/backend/models/test/test_modeling.py
+++ b/products/data_warehouse/backend/models/test/test_modeling.py
@@ -15,16 +15,12 @@ from products.data_modeling.backend.facade.modeling import (
RESOLUTION_SOURCE_LINEAGE,
RESOLUTION_SOURCE_MATERIALIZATION,
BoundedResolver,
- DataWarehouseModelPath,
- NodeType,
ResolutionCycleError,
ResolutionDepthExceededError,
ResolutionTimeoutError,
get_parents_from_model_query,
)
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
-from products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSchema, ExternalDataSource
-from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
GET_PARENTS_TEST_CASES = [
("select events.*, persons.* from events, persons", {"events", "persons"}),
@@ -160,7 +156,7 @@ GET_PARENTS_TEST_CASES = [
]
-class TestModelPath(BaseTest):
+class TestGetParentsFromModelQuery(BaseTest):
@parameterized.expand(GET_PARENTS_TEST_CASES)
def test_get_parents_from_model_query(self, model_query: str, parents: set[str]):
model_name = "test_model"
@@ -171,342 +167,6 @@ class TestModelPath(BaseTest):
with pytest.raises(QueryError, match="Unknown table"):
get_parents_from_model_query(self.team, "test_model", "select * from some_random_view")
- def test_create_from_static_query(self):
- """Test creation of a model path from a query that returns a static set of rows."""
- query = "SELECT 1 AS a, 2 AS b, NOW() AS c"
- saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model",
- query={"query": query},
- )
-
- model_paths = DataWarehouseModelPath.objects.create_from_saved_query(saved_query)
-
- paths = [model_path.path for model_path in model_paths]
- self.assertEqual(len(paths), 1)
- self.assertIn([saved_query.id.hex], paths)
-
- def test_create_from_posthog_root_nodes_query(self):
- """Test creation of a model path from a query that reads from PostHog root tables."""
- query = """\
- select
- events.event,
- persons.properties
- from events
- left join persons on events.person_id = persons.id
- where events.event = 'login' and events.distinct_id != 'some_distinct_id'
- """
- saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model",
- query={"query": query},
- )
-
- model_paths = DataWarehouseModelPath.objects.create_from_saved_query(saved_query)
- paths = [model_path.path for model_path in model_paths]
-
- self.assertEqual(len(paths), 2)
- self.assertIn(["events", saved_query.id.hex], paths)
- self.assertIn(["persons", saved_query.id.hex], paths)
-
- def test_update_paths_for_posthog_namespaced_table_query(self):
- """`posthog.*`-namespaced tables (e.g. ai_events) must resolve as valid model-path parents on
- the UPDATE path too. `update_paths_from_query` keys off `get_posthog_table_names()`, which omits
- namespaced tables — so re-saving/re-materializing a view over `posthog.ai_events` raised
- UnknownParentError until the hidden table names were unioned in. (The create path already
- resolves them via `get_table()`.)"""
- query = "SELECT trace_id FROM posthog.ai_events"
- saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model",
- query={"query": query},
- )
-
- # Create path seeds the initial paths (it resolves namespaced tables via get_table()).
- DataWarehouseModelPath.objects.create_from_saved_query(saved_query)
-
- # Update path is hit whenever an existing view is re-saved or re-materialized.
- DataWarehouseModelPath.objects.update_from_saved_query(saved_query)
-
- # `posthog.ai_events` is stored as two ltree labels (LabelTreeField splits on "."), exactly
- # as the create path stores it — the update path must converge on the same single path.
- paths = [mp.path for mp in DataWarehouseModelPath.objects.filter(team=self.team, saved_query=saved_query)]
- self.assertEqual(len(paths), 1)
- self.assertIn(["posthog", "ai_events", saved_query.id.hex], paths)
-
- def test_create_from_warehouse_table_old_notation_nodes_query(self):
- """Test creation of a model path from a query that reads from a managed source using old notation."""
-
- source = ExternalDataSource.objects.create(team=self.team, source_type=ExternalDataSourceType.STRIPE)
- table = DataWarehouseTable.objects.create(team=self.team, name="stripe_invoice", external_data_source=source)
- ExternalDataSchema.objects.create(team=self.team, name="Invoice", source=source, table=table)
-
- query = """\
- select *
- from stripe_invoice
- """
- saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model",
- query={"query": query},
- )
-
- model_paths = DataWarehouseModelPath.objects.create_from_saved_query(saved_query)
- paths = [model_path.path for model_path in model_paths]
-
- self.assertEqual(len(paths), 1)
- self.assertIn([table.id.hex, saved_query.id.hex], paths)
-
- def test_create_from_warehouse_table_new_notation_nodes_query(self):
- """Test creation of a model path from a query that reads from a managed source using new notation."""
-
- source = ExternalDataSource.objects.create(team=self.team, source_type=ExternalDataSourceType.STRIPE)
- table = DataWarehouseTable.objects.create(team=self.team, name="stripe_invoice", external_data_source=source)
- ExternalDataSchema.objects.create(team=self.team, name="Invoice", source=source, table=table)
-
- query = """\
- select *
- from stripe.invoice
- """
- saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model",
- query={"query": query},
- )
-
- model_paths = DataWarehouseModelPath.objects.create_from_saved_query(saved_query)
- paths = [model_path.path for model_path in model_paths]
-
- self.assertEqual(len(paths), 1)
- self.assertIn([table.id.hex, saved_query.id.hex], paths)
-
- def test_create_from_table_functions_root_nodes_query(self):
- """Table functions like numbers() are not real parents — they produce root nodes."""
- query = "select * from numbers(10)"
- saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model",
- query={"query": query},
- )
-
- model_paths = DataWarehouseModelPath.objects.create_from_saved_query(saved_query)
- paths = [model_path.path for model_path in model_paths]
-
- self.assertEqual(len(paths), 1)
- self.assertIn([saved_query.id.hex], paths)
-
- def test_create_from_existing_path(self):
- """Test creation of a model path from a query that reads from another query."""
- parent_query = """\
- select
- events.event,
- persons.properties
- from events
- left join persons on events.person_id = persons.id
- where events.event = 'login' and events.distinct_id != 'some_distinct_id'
- """
- parent_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model",
- query={"query": parent_query},
- )
- child_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model_child",
- query={"query": "select * from my_model as my_other_model"},
- )
-
- parent_model_paths = DataWarehouseModelPath.objects.create_from_saved_query(parent_saved_query)
- child_model_paths = DataWarehouseModelPath.objects.create_from_saved_query(child_saved_query)
-
- parent_paths = [model_path.path for model_path in parent_model_paths]
- child_paths = [model_path.path for model_path in child_model_paths]
-
- self.assertEqual(len(parent_paths), 2)
- self.assertIn(["events", parent_saved_query.id.hex], parent_paths)
- self.assertIn(["persons", parent_saved_query.id.hex], parent_paths)
-
- self.assertEqual(len(child_paths), 2)
- self.assertIn(["events", parent_saved_query.id.hex, child_saved_query.id.hex], child_paths)
- self.assertIn(["persons", parent_saved_query.id.hex, child_saved_query.id.hex], child_paths)
-
- def test_update_path_from_saved_query(self):
- """Test update of a model path from a query that reads from another query."""
- parent_query = """\
- select
- events.event,
- persons.properties
- from events
- left join persons on events.person_id = persons.id
- where events.event = 'login' and events.distinct_id != 'some_distinct_id'
- """
- parent_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model",
- query={"query": parent_query},
- )
- child_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model_child",
- query={"query": "select * from my_model as my_other_model"},
- )
- grand_child_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model_grand_child",
- query={"query": "select * from my_model_child"},
- )
-
- DataWarehouseModelPath.objects.create_from_saved_query(parent_saved_query)
- DataWarehouseModelPath.objects.create_from_saved_query(child_saved_query)
- DataWarehouseModelPath.objects.create_from_saved_query(grand_child_saved_query)
-
- child_saved_query.query = {"query": "select * from events as my_other_model"}
- child_saved_query.save()
- DataWarehouseModelPath.objects.update_from_saved_query(child_saved_query)
-
- child_refreshed_model_paths = DataWarehouseModelPath.objects.filter(
- team=self.team, saved_query=child_saved_query
- ).all()
- child_paths = [model_path.path for model_path in child_refreshed_model_paths]
- grand_child_refreshed_model_paths = DataWarehouseModelPath.objects.filter(
- team=self.team, saved_query=grand_child_saved_query
- ).all()
- grand_child_paths = [model_path.path for model_path in grand_child_refreshed_model_paths]
-
- self.assertEqual(len(child_paths), 1)
- self.assertIn(["events", child_saved_query.id.hex], child_paths)
- self.assertEqual(len(grand_child_paths), 1)
- self.assertIn(["events", child_saved_query.id.hex, grand_child_saved_query.id.hex], grand_child_paths)
-
- def test_get_dag(self):
- """Test the generation of a DAG with a couple simple models."""
- parent_query = """\
- select
- events.event,
- persons.properties
- from events
- left join persons on events.person_id = persons.id
- where events.event = 'login' and events.distinct_id != 'some_distinct_id'
- """
- parent_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model",
- query={"query": parent_query},
- )
- child_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model_child",
- query={"query": "select * from my_model as my_other_model"},
- )
-
- DataWarehouseModelPath.objects.create_from_saved_query(parent_saved_query)
- DataWarehouseModelPath.objects.create_from_saved_query(child_saved_query)
-
- dag = DataWarehouseModelPath.objects.get_dag(team=self.team)
-
- self.assertIn((parent_saved_query.id.hex, child_saved_query.id.hex), dag.edges)
- self.assertIn(("events", parent_saved_query.id.hex), dag.edges)
- self.assertIn(("persons", parent_saved_query.id.hex), dag.edges)
- self.assertEqual(len(dag.edges), 3)
-
- self.assertIn((child_saved_query.id.hex, NodeType.SAVED_QUERY), dag.nodes)
- self.assertIn((parent_saved_query.id.hex, NodeType.SAVED_QUERY), dag.nodes)
- self.assertIn(("events", NodeType.POSTHOG), dag.nodes)
- self.assertIn(("persons", NodeType.POSTHOG), dag.nodes)
- self.assertEqual(len(dag.nodes), 4)
-
- def test_update_child_when_parent_has_multiple_paths_does_not_crash(self):
- """Test updating a child model when its parent has multiple paths doesn't crash.
-
- This tests the fix for CardinalityViolation that occurred when a parent
- had multiple paths (e.g., from selecting from multiple root tables).
- The cross join in UPDATE_PATHS_QUERY would produce duplicate IDs,
- which PostgreSQL rejected. Using DISTINCT ON (id) fixes this.
-
- The UPDATE_PATHS_QUERY only updates descendant paths (where the child
- has something after it), so we need a grandchild to trigger the cross join.
- """
- parent_query = """\
- select
- events.event,
- persons.properties
- from events
- left join persons on events.person_id = persons.id
- """
- parent_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="parent",
- query={"query": parent_query},
- )
- child_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="child",
- query={"query": "select * from parent"},
- )
- grandchild_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="grandchild",
- query={"query": "select * from child"},
- )
-
- DataWarehouseModelPath.objects.create_from_saved_query(parent_saved_query)
- DataWarehouseModelPath.objects.create_from_saved_query(child_saved_query)
- DataWarehouseModelPath.objects.create_from_saved_query(grandchild_saved_query)
-
- parent_paths = list(
- DataWarehouseModelPath.objects.filter(saved_query=parent_saved_query).values_list("path", flat=True)
- )
- self.assertEqual(len(parent_paths), 2)
- child_saved_query.query = {"query": "select event, properties from parent where event = 'login'"}
- child_saved_query.save()
-
- # this would raise ProgrammingError with "CardinalityViolation"" before we were using DISTINCT ON (id)
- DataWarehouseModelPath.objects.update_from_saved_query(child_saved_query)
- grandchild_paths = list(
- DataWarehouseModelPath.objects.filter(saved_query=grandchild_saved_query).values_list("path", flat=True)
- )
- self.assertGreaterEqual(len(grandchild_paths), 1)
- # verify the lineage is still correct
- self.assertEqual(grandchild_paths[0][-1], grandchild_saved_query.id.hex)
- self.assertEqual(grandchild_paths[0][-2], child_saved_query.id.hex)
- self.assertEqual(grandchild_paths[0][-3], parent_saved_query.id.hex)
-
- def test_creating_cycles_via_updates_raises_exception(self):
- """Test cycles cannot be created just by updating queries that select from each other."""
- parent_query = """\
- select
- events.event,
- persons.properties
- from events
- left join persons on events.person_id = persons.id
- where events.event = 'login' and events.distinct_id != 'some_distinct_id'
- """
- parent_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model",
- query={"query": parent_query},
- )
- child_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model_child",
- query={"query": "select * from my_model"},
- )
- grand_child_saved_query = DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="my_model_grand_child",
- query={"query": "select * from my_model_child"},
- )
-
- DataWarehouseModelPath.objects.create_from_saved_query(parent_saved_query)
- DataWarehouseModelPath.objects.create_from_saved_query(child_saved_query)
- DataWarehouseModelPath.objects.create_from_saved_query(grand_child_saved_query)
-
- child_saved_query.query = {"query": "select * from my_model union all select * from my_model_grand_child"}
- child_saved_query.save()
-
- with pytest.raises(ResolutionCycleError, match="[Cc]ircular dependency"):
- DataWarehouseModelPath.objects.update_from_saved_query(child_saved_query)
-
class TestBoundedResolver(BaseTest):
def _resolve(
diff --git a/products/data_warehouse/backend/models/test/test_saved_query.py b/products/data_warehouse/backend/models/test/test_saved_query.py
index 2b4c3d6cb071..377144697fcb 100644
--- a/products/data_warehouse/backend/models/test/test_saved_query.py
+++ b/products/data_warehouse/backend/models/test/test_saved_query.py
@@ -4,8 +4,6 @@ from types import SimpleNamespace
from posthog.test.base import BaseTest, ClickhouseTestMixin
from unittest.mock import patch
-from django.db.models.query import QuerySet as DjangoQuerySet
-
from parameterized import parameterized
from posthog.schema import CachedHogQLQueryResponse, HogQLQuery
@@ -17,20 +15,11 @@ from posthog.constants import AvailableFeature
from posthog.hogql_queries.hogql_query_runner import HogQLQueryRunner
from posthog.hogql_queries.query_runner import ExecutionMode
-from products.data_modeling.backend.facade.modeling import DataWarehouseModelPath
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.warehouse_sources.backend.facade.models import DataWarehouseCredential, DataWarehouseTable
class TestRevertMaterialization(BaseTest):
- """Tests for DataWarehouseSavedQuery.revert_materialization.
-
- The method runs three DB operations (table soft-delete, saved_query save,
- model paths delete) inside one transaction. If any of them fails, every
- piece of state must be left as it was, so the next retry can converge on a
- consistent state.
- """
-
def setUp(self):
super().setUp()
self.credential = DataWarehouseCredential.objects.create(
@@ -55,15 +44,8 @@ class TestRevertMaterialization(BaseTest):
sync_frequency_interval=timedelta(hours=1),
status=DataWarehouseSavedQuery.Status.COMPLETED,
)
- DataWarehouseModelPath.objects.create(
- team=self.team,
- saved_query=self.saved_query,
- path=["posthog_events", self.saved_query.id.hex],
- )
def _assert_state_unchanged(self) -> None:
- """A rollback must leave every piece of state (saved_query, table, model paths)
- exactly as it was before revert_materialization was called."""
self.saved_query.refresh_from_db()
self.assertTrue(self.saved_query.is_materialized)
self.assertEqual(self.saved_query.sync_frequency_interval, timedelta(hours=1))
@@ -73,8 +55,6 @@ class TestRevertMaterialization(BaseTest):
self.table.refresh_from_db()
self.assertFalse(self.table.deleted)
- self.assertTrue(DataWarehouseModelPath.objects.filter(team=self.team, saved_query=self.saved_query).exists())
-
def test_state_cleared_when_all_operations_succeed(self):
self.saved_query.revert_materialization()
@@ -87,22 +67,9 @@ class TestRevertMaterialization(BaseTest):
self.table.refresh_from_db()
self.assertTrue(self.table.deleted)
- self.assertFalse(DataWarehouseModelPath.objects.filter(team=self.team, saved_query=self.saved_query).exists())
-
- def test_rollback_when_model_path_delete_raises(self):
- """The last DB op raising must roll back the table soft_delete and the
- saved_query save that ran earlier in the same block."""
- original_delete = DjangoQuerySet.delete
-
- def failing_delete(self_qs, *args, **kwargs):
- # Only raise for DataWarehouseModelPath querysets so that test fixture
- # cleanup and any unrelated delete calls still work normally.
- if self_qs.model is DataWarehouseModelPath:
- raise RuntimeError("model path delete failed")
- return original_delete(self_qs, *args, **kwargs)
-
- with patch.object(DjangoQuerySet, "delete", failing_delete):
- with self.assertRaisesRegex(RuntimeError, "model path delete failed"):
+ def test_rollback_when_saved_query_save_raises(self):
+ with patch.object(DataWarehouseSavedQuery, "save", side_effect=RuntimeError("saved query save failed")):
+ with self.assertRaisesRegex(RuntimeError, "saved query save failed"):
self.saved_query.revert_materialization()
self._assert_state_unchanged()
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/editing.py b/products/data_warehouse/backend/presentation/views/saved_query/editing.py
index 63df70a7481c..6597a80ebaa9 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/editing.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/editing.py
@@ -374,14 +374,6 @@ class DataWarehouseSavedQuerySerializer(
view.save()
if has_description:
self._write_view_description(view, description)
- try:
- view.setup_model_paths()
- except Exception:
- # For now, do not fail saved query creation if we cannot model-ize it.
- # Later, after bugs and errors have been ironed out, we may tie these two
- # closer together.
- logger.exception("Failed to create model path when creating view %s", view.name)
-
team = Team.objects.get(id=view.team_id)
log_activity(
@@ -550,12 +542,6 @@ class DataWarehouseSavedQuerySerializer(
view.status = DataWarehouseSavedQuery.Status.MODIFIED
view.save()
- try:
- view.setup_model_paths()
- except Exception as e:
- capture_exception(e)
- logger.exception("Failed to update model path when updating view %s", view.name)
-
team = Team.objects.get(id=view.team_id)
changes = changes_between("DataWarehouseSavedQuery", previous=before_update, current=view)
diff --git a/products/data_warehouse/backend/tests/api/test_misc_warehouse_viewsets_access_control.py b/products/data_warehouse/backend/tests/api/test_misc_warehouse_viewsets_access_control.py
index 25b5100d171c..6f76c82d5341 100644
--- a/products/data_warehouse/backend/tests/api/test_misc_warehouse_viewsets_access_control.py
+++ b/products/data_warehouse/backend/tests/api/test_misc_warehouse_viewsets_access_control.py
@@ -1,11 +1,3 @@
-"""Access control coverage for the non-core data warehouse viewsets.
-
-Covers managed viewsets, the data_warehouse stats/provisioning endpoints, model paths,
-data modeling jobs, and lineage — each gated under warehouse_view (reads) or
-warehouse_view:write (write actions). Verifies viewer/editor/none rejection and
-acceptance for the endpoints that actually get traffic.
-"""
-
from datetime import timedelta
import pytest
diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.py
index bf99b3fabbc3..8c0dd98199ee 100644
--- a/products/data_warehouse/backend/tests/api/test_saved_query.py
+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py
@@ -22,7 +22,6 @@ from posthog.models import ActivityLog
from posthog.models.scoping import team_scope
from products.data_modeling.backend.facade.api import UnsatisfiableFrequencyError, mark_node_suspended, suspension_state
-from products.data_modeling.backend.facade.modeling import DataWarehouseModelPath
from products.data_modeling.backend.facade.models import (
DAG,
DataModelingJob,
@@ -1480,58 +1479,6 @@ class TestSavedQuery(APIBaseTest):
)
self.assertEqual(saved_view_2_response.status_code, 201, saved_view_2_response.content)
- def test_create_with_saved_query(self):
- response = self.client.post(
- f"/api/environments/{self.team.id}/warehouse_saved_queries/",
- {
- "name": "event_view",
- "query": {
- "kind": "HogQLQuery",
- "query": "select event as event from events",
- },
- },
- )
-
- self.assertEqual(response.status_code, 201, response.content)
- saved_query_id = response.json()["id"]
- paths = list(DataWarehouseModelPath.objects.filter(saved_query_id=saved_query_id).all())
- self.assertEqual(len(paths), 1)
- self.assertEqual(["events", uuid.UUID(saved_query_id).hex], paths[0].path)
-
- def test_create_with_nested_saved_query(self):
- response_1 = self.client.post(
- f"/api/environments/{self.team.id}/warehouse_saved_queries/",
- {
- "name": "event_view",
- "query": {
- "kind": "HogQLQuery",
- "query": "select event as event from events",
- },
- },
- )
- self.assertEqual(response_1.status_code, 201, response_1.content)
-
- response_2 = self.client.post(
- f"/api/environments/{self.team.id}/warehouse_saved_queries/",
- {
- "name": "event_view_2",
- "query": {
- "kind": "HogQLQuery",
- "query": "select event as event from event_view",
- },
- },
- )
- self.assertEqual(response_2.status_code, 201, response_1.content)
-
- saved_query_id_hex_1 = uuid.UUID(response_1.json()["id"]).hex
- saved_query_id_hex_2 = uuid.UUID(response_2.json()["id"]).hex
-
- paths = [model_path.path for model_path in DataWarehouseModelPath.objects.all()]
- self.assertEqual(len(paths), 3)
- self.assertIn(["events"], paths)
- self.assertIn(["events", saved_query_id_hex_1], paths)
- self.assertIn(["events", saved_query_id_hex_1, saved_query_id_hex_2], paths)
-
def test_ancestors(self):
query = """\
select
@@ -2166,8 +2113,6 @@ class TestSavedQuery(APIBaseTest):
db_saved_query.table = mock_table
db_saved_query.save()
- DataWarehouseModelPath.objects.create(team=self.team, path=[mock_table.id.hex, db_saved_query.id.hex])
-
response = self.client.post(
f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query_id}/revert_materialization",
)
@@ -2186,13 +2131,6 @@ class TestSavedQuery(APIBaseTest):
mock_table.refresh_from_db()
self.assertTrue(mock_table.deleted)
- self.assertEqual(
- DataWarehouseModelPath.objects.filter(
- team=self.team, path__lquery=f"*{{1,}}.{db_saved_query.id.hex}"
- ).count(),
- 0,
- )
-
def test_create_with_existing_name(self):
DataWarehouseTable.objects.create(
team=self.team, name="some_event_table", format="Parquet", url_pattern="s3://bucket/path"
diff --git a/products/endpoints/backend/tests/test_endpoint_materialization.py b/products/endpoints/backend/tests/test_endpoint_materialization.py
index 5949acabc44c..67b94f8356c4 100644
--- a/products/endpoints/backend/tests/test_endpoint_materialization.py
+++ b/products/endpoints/backend/tests/test_endpoint_materialization.py
@@ -26,7 +26,6 @@ from posthog.models.team.team_revenue_analytics_config import TeamRevenueAnalyti
from posthog.sync import database_sync_to_async
from products.data_modeling.backend.facade.api import UnsatisfiableFrequencyError, get_declared_target
-from products.data_modeling.backend.facade.modeling import DataWarehouseModelPath
from products.data_modeling.backend.facade.models import DAG, DataModelingJob, DataWarehouseSavedQuery, Node
from products.endpoints.backend.insight_transformers import transform_materialized_insight_response
from products.endpoints.backend.logic.execution import EndpointExecutionService
@@ -2472,8 +2471,6 @@ class TestEndpointMaterializationTemporal:
saved_query.columns = await sync_to_async(saved_query.get_columns)()
await sync_to_async(saved_query.save)()
- await database_sync_to_async(DataWarehouseModelPath.objects.create_from_saved_query)(saved_query)
-
# Link saved_query to version instead of endpoint
version.saved_query = saved_query
await sync_to_async(version.save)()
diff --git a/products/managed_warehouse/backend/table_binding.py b/products/managed_warehouse/backend/table_binding.py
index f2ab9b205d45..5c0a97ba19f8 100644
--- a/products/managed_warehouse/backend/table_binding.py
+++ b/products/managed_warehouse/backend/table_binding.py
@@ -39,19 +39,8 @@ def bind_tables_to_ducklake(database: Any, team_id: int) -> None:
def get_data_modeling_table_names(team_id: int, saved_query_ids: Sequence[UUID]) -> dict[UUID, str]:
- from products.data_modeling.backend.facade.modeling import DataWarehouseModelPath
-
- labels = {
- saved_query_id: path[-1]
- for saved_query_id, path in DataWarehouseModelPath.objects.filter(
- team_id=team_id, saved_query_id__in=saved_query_ids
- ).values_list("saved_query_id", "path")
- if path
- }
return {
- saved_query_id: ducklake_data_modeling_table_name(
- labels.get(saved_query_id, saved_query_id.hex), saved_query_id.hex
- )
+ saved_query_id: ducklake_data_modeling_table_name(saved_query_id.hex, saved_query_id.hex)
for saved_query_id in saved_query_ids
}
diff --git a/products/managed_warehouse/backend/tests/test_trino_compiler.py b/products/managed_warehouse/backend/tests/test_trino_compiler.py
index 8405d22cc3e6..658b4db83ff1 100644
--- a/products/managed_warehouse/backend/tests/test_trino_compiler.py
+++ b/products/managed_warehouse/backend/tests/test_trino_compiler.py
@@ -21,7 +21,6 @@ from posthog.hogql.transforms.trino.manifest import (
from posthog.models import Organization, Team
from posthog.schema_enums import DatabaseSerializedFieldType
-from products.data_modeling.backend.facade.modeling import DataWarehouseModelPath
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.managed_warehouse.backend.facade.contracts import (
ManagedWarehouseTableNames,
@@ -515,7 +514,7 @@ def test_build_trino_table_locators_uses_provisioned_names_and_canonical_source_
url_pattern="https://bucket.s3.amazonaws.com/models/orders/*.parquet",
)
saved_query_id = UUID("32345678-1234-5678-1234-567812345678")
- saved_query = DataWarehouseSavedQuery.objects.create(
+ DataWarehouseSavedQuery.objects.create(
id=saved_query_id,
team=team,
name="orders_model",
@@ -523,11 +522,6 @@ def test_build_trino_table_locators_uses_provisioned_names_and_canonical_source_
table=model_table,
is_materialized=True,
)
- DataWarehouseModelPath.objects.create(
- team=team,
- saved_query=saved_query,
- path=["legacy_orders_model"],
- )
source = ExternalDataSource.objects.create(
team=team,
source_id="source_id",
@@ -578,7 +572,7 @@ def test_build_trino_table_locators_uses_provisioned_names_and_canonical_source_
"orders_model": (
"org_catalog",
f"posthog_data_modeling_team_{team.pk}",
- "legacy_orders_model",
+ f"model_{saved_query_id.hex}",
),
"myprefix_stripe_customers": ("org_catalog", "imports_production", "stripe_myprefix_customers"),
"stripe.myprefix.customers": ("org_catalog", "imports_production", "stripe_myprefix_customers"),
diff --git a/products/managed_warehouse/backend/tests/test_trino_materialization.py b/products/managed_warehouse/backend/tests/test_trino_materialization.py
index 5ac6150079f7..8a2123b7e0a3 100644
--- a/products/managed_warehouse/backend/tests/test_trino_materialization.py
+++ b/products/managed_warehouse/backend/tests/test_trino_materialization.py
@@ -11,7 +11,6 @@ from posthog.schema import HogQLQuery
from posthog.models import Team
-from products.data_modeling.backend.facade.modeling import DataWarehouseModelPath
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.managed_warehouse.backend.facade.client import execute_trino_shadow_materialization
from products.managed_warehouse.backend.facade.contracts import (
@@ -193,13 +192,12 @@ class TestTrinoShadowMaterialization(BaseTest):
compile_query.assert_not_called()
connect.assert_not_called()
- @parameterized.expand([(None,), ("legacy_orders",)])
- def test_materialized_dependency_matches_downstream_locator(self, model_label: str | None) -> None:
+ def test_materialized_dependency_matches_downstream_locator(self) -> None:
backing_table = DataWarehouseTable.objects.create(
team=self.team, name="orders", format="Parquet", url_pattern="https://example.com/orders/*.parquet"
)
- saved_query = DataWarehouseSavedQuery.objects.create(
+ DataWarehouseSavedQuery.objects.create(
id=self.saved_query_id,
team=self.team,
name="orders",
@@ -207,8 +205,6 @@ class TestTrinoShadowMaterialization(BaseTest):
is_materialized=True,
table=backing_table,
)
- if model_label:
- DataWarehouseModelPath.objects.create(team=self.team, saved_query=saved_query, path=[model_label])
with (
patch(
"products.managed_warehouse.backend.trino_materialization.compile_hogql_to_trino_sql",
@@ -231,7 +227,7 @@ class TestTrinoShadowMaterialization(BaseTest):
),
)
assert locators["orders"] == (self.connection.catalog, written.schema_name, written.table_name)
- assert written.table_name == (model_label or f"model_{self.saved_query_id.hex}")
+ assert written.table_name == f"model_{self.saved_query_id.hex}"
class _ScriptedCursor:
diff --git a/products/model_crossing_uses_baseline.txt b/products/model_crossing_uses_baseline.txt
index d469dbc27a11..d1779103ecf9 100644
--- a/products/model_crossing_uses_baseline.txt
+++ b/products/model_crossing_uses_baseline.txt
@@ -459,7 +459,6 @@ posthog.User dashboards.Text.last_modified_by reverse-accessor(modified_text_til
posthog.User data_modeling.DAG.created_by reverse-accessor(dag_set) 1
posthog.User data_modeling.DataModelingJob.created_by reverse-accessor(datamodelingjob_set) 1
posthog.User data_modeling.DataWarehouseManagedViewSet.created_by reverse-accessor(datawarehousemanagedviewset_set) 1
-posthog.User data_modeling.DataWarehouseModelPath.created_by reverse-accessor(datawarehousemodelpath_set) 1
posthog.User data_modeling.DataWarehouseSavedQuery.created_by reverse-accessor(datawarehousesavedquery_set) 1
posthog.User data_modeling.DataWarehouseSavedQueryDraft.created_by reverse-accessor(datawarehousesavedquerydraft_set) 1
posthog.User data_modeling.Edge.created_by reverse-accessor(edge_set) 1
@@ -759,7 +758,6 @@ warehouse_sources.DataWarehouseTable products.data_catalog.evals.seeders write(c
warehouse_sources.DataWarehouseTable products.data_modeling.backend.logic.node_frequency instance-many(exclude) 1
warehouse_sources.DataWarehouseTable products.data_modeling.backend.logic.saved_query_dag_sync instance-single 2
warehouse_sources.DataWarehouseTable products.data_modeling.backend.management.commands.cleanup_orphaned_matview_tables instance-many(filter) 1
-warehouse_sources.DataWarehouseTable products.data_modeling.backend.models.modeling instance-single 4
warehouse_sources.DataWarehouseTable products.data_warehouse.backend.direct_clickhouse write(create) 1
warehouse_sources.DataWarehouseTable products.data_warehouse.backend.direct_motherduck write(create) 1
warehouse_sources.DataWarehouseTable products.data_warehouse.backend.direct_mysql write(create) 1