:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): run a model on trino from the saved query run endpoint (#124913)
Jovan Sakovic8dccb500bd5f
diff --git a/posthog/settings/web.py b/posthog/settings/web.pyindex bfd50d65204b..e31ed356106b 100644--- a/posthog/settings/web.py+++ b/posthog/settings/web.py@@ -664,6 +664,19 @@ SPECTACULAR_SETTINGS = {# The published name is already derived by a different choice set, so the# entry holds this one apart."SlackSummaryCadenceEnum": ["daily", "weekly", "monthly"],+ # Keeps `EngineEnum` for the warehouse source engines. The saved-query run endpoint+ # also has an `engine` field, which would otherwise rename this one.+ "EngineEnum": [+ "duckdb",+ "postgres",+ "mysql",+ "snowflake",+ "redshift",+ "clickhouse",+ "motherduck",+ "trino",+ None,+ ],# signals' report-metric role; AutoresearchModel.Role also sits on a field named `role`."RoleEnum": ["primary", "supporting"],# Keeps the name `CodeEnum` for the dataset conflict codes of ai_observability. A second field nameddiff --git a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.pyindex ccb502c6cab7..16b56b274f81 100644--- a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py+++ b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py@@ -173,7 +173,7 @@ def _is_managed_warehouse_shadow_flag_enabled(team: Team) -> bool:return False-def _is_managed_warehouse_shadow_enabled(team: Team) -> bool:+def is_managed_warehouse_shadow_enabled(team: Team) -> bool:if not _is_managed_warehouse_shadow_flag_enabled(team):return False@@ -220,7 +220,7 @@ def _get_shadow_input_objects(inputs: ManagedWarehouseShadowInputs) -> _ManagedW@database_sync_to_async_pooldef _check_managed_warehouse_shadow_eligibility(inputs: ManagedWarehouseShadowEligibilityInputs) -> bool:objects = _load_shadow_objects(team_id=inputs.team_id, dag_id=inputs.dag_id, node_id=inputs.node_id)- return _is_managed_warehouse_shadow_enabled(objects.team)+ return is_managed_warehouse_shadow_enabled(objects.team)async def _check_managed_warehouse_shadow_enabled_activity(team_id: int) -> bool:@@ -236,7 +236,7 @@ async def check_managed_warehouse_shadow_enabled_activity(team_id: int) -> bool:@database_sync_to_async_pooldef _check_team_managed_warehouse_shadow_eligibility(team_id: int) -> bool:- return _is_managed_warehouse_shadow_enabled(Team.objects.get(id=team_id))+ return is_managed_warehouse_shadow_enabled(Team.objects.get(id=team_id))@activity.defndiff --git a/posthog/temporal/data_modeling/workflows/execute_dag.py b/posthog/temporal/data_modeling/workflows/execute_dag.pyindex 8d2d590fe856..4b150bc10bb8 100644--- a/posthog/temporal/data_modeling/workflows/execute_dag.py+++ b/posthog/temporal/data_modeling/workflows/execute_dag.py@@ -77,6 +77,20 @@ def trino_shadow_workflow_id(workflow_id: str, dag_id: str) -> str:return f"execute-dag-trino-{dag_id}"+def manual_trino_workflow_id(dag_id: str) -> str:+ """The fixed id of a Trino run that a person starts by hand.++ It shares the `execute-dag-trino-{dag_id}` prefix with the scheduled shadows, so one+ visibility query finds every Trino run of the DAG.+ """+ return f"execute-dag-trino-{dag_id}-manual"+++def trino_runs_of_dag_query(dag_id: str) -> str:+ prefix = f"execute-dag-trino-{dag_id}"+ return f'WorkflowId >= "{prefix}" AND WorkflowId < "{prefix}~" AND ExecutionStatus = "Running"'++async def _kill_switch_child_limit(team_id: int) -> int:try:level = await temporalio.workflow.execute_activity(@@ -166,6 +180,7 @@ class ExecuteDAGInputs:all materializable nodes in the DAG will be processed.shadow: a Trino shadow of a ClickHouse run. Its failures stay in job rows and metrics,so it runs no data quality checks and sends no failure notifications.+ manually_triggered_by_id: the user who asked for this run, when a person did."""team_id: int@@ -176,6 +191,7 @@ class ExecuteDAGInputs:# Old workflow payloads contain this field, so removing it would prevent replay after deployment.duckgres_only: bool = Falseshadow: bool = False+ manually_triggered_by_id: int | None = None@propertydef properties_to_log(self) -> dict:@@ -510,6 +526,7 @@ class ExecuteDAGWorkflow(PostHogWorkflow):node_id=node_id,managed_warehouse_only=managed_warehouse_only,dangerously_execute_raw_sql=inputs.dangerously_execute_raw_sql,+ manually_triggered_by_id=inputs.manually_triggered_by_id,skip_trino=bool(blocked_trino_upstreams),)trino_skip = (diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.pyindex 76a6e42165e7..086cecb60265 100644--- a/products/data_modeling/backend/facade/api.py+++ b/products/data_modeling/backend/facade/api.py@@ -37,6 +37,7 @@ _LAZY = {"is_saved_query_on_v2_schedule": "logic.node_materialization","materialize_saved_query": "logic.node_materialization","run_related_node_materializations": "logic.node_materialization",+ "run_trino_node_materializations": "logic.node_materialization","run_saved_query_materialization": "logic.node_materialization","get_materialized_table_uri": "logic.saved_query_reads","get_node_ids_for_saved_queries": "logic.saved_query_reads",@@ -63,6 +64,9 @@ _LAZY = {"completed_full_refresh_workflow_ids": "logic.saved_query_reads","MaterializationFailedError": "facade.contracts","MaterializationForbiddenError": "facade.contracts",+ "MaterializationAlreadyRunningError": "facade.contracts",+ "MaterializationEngine": "facade.contracts",+ "ManagedWarehouseUnavailableError": "facade.contracts","MaterializationRunDirection": "facade.contracts","MaterializationRefusedError": "facade.contracts","IncrementalConfigRefusedError": "facade.contracts",diff --git a/products/data_modeling/backend/facade/contracts.py b/products/data_modeling/backend/facade/contracts.pyindex b61bfe5ce164..1795f3f223eb 100644--- a/products/data_modeling/backend/facade/contracts.py+++ b/products/data_modeling/backend/facade/contracts.py@@ -116,9 +116,22 @@ class MaterializationRunDirection(LabeledStrEnum):DOWNSTREAM = "downstream", "Downstream"+class MaterializationEngine(LabeledStrEnum):+ CLICKHOUSE = "clickhouse", "ClickHouse"+ TRINO = "trino", "Trino"++class MaterializationForbiddenError(Exception):pass+class ManagedWarehouseUnavailableError(MaterializationForbiddenError):+ pass+++class MaterializationAlreadyRunningError(Exception):+ pass++class MaterializationFailedError(Exception):passdiff --git a/products/data_modeling/backend/logic/node_materialization.py b/products/data_modeling/backend/logic/node_materialization.pyindex 9ee8b647698b..2bfcd9349ef8 100644--- a/products/data_modeling/backend/logic/node_materialization.py+++ b/products/data_modeling/backend/logic/node_materialization.py@@ -4,24 +4,40 @@ from datetime import timedeltafrom uuid import UUID, uuid4from django.conf import settings+from django.db import transactionimport structlogfrom temporalio.common import RetryPolicy, WorkflowIDConflictPolicy, WorkflowIDReusePolicy+from temporalio.exceptions import WorkflowAlreadyStartedErrorfrom posthog.hogql.database.database import Databasefrom posthog.models import Userfrom posthog.rbac.query_access import assert_user_can_read_queryfrom posthog.temporal.common.client import sync_connect-from posthog.temporal.data_modeling.workflows.execute_dag import ExecuteDAGInputs+from posthog.temporal.data_modeling.activities.materialize_view_managed_warehouse import (+ is_managed_warehouse_shadow_enabled,+)+from posthog.temporal.data_modeling.workflows.execute_dag import (+ TRINO_SHADOW_RUN_TIMEOUT,+ ExecuteDAGInputs,+ manual_trino_workflow_id,+ trino_runs_of_dag_query,+)from posthog.temporal.data_modeling.workflows.materialize_view import MaterializeViewWorkflowInputsfrom products.access_control.backend.facade.user_access_control import UserAccessControl-from products.data_modeling.backend.facade.contracts import MaterializationForbiddenError, MaterializationRunDirection+from products.data_modeling.backend.facade.contracts import (+ ManagedWarehouseUnavailableError,+ MaterializationAlreadyRunningError,+ MaterializationForbiddenError,+ MaterializationRunDirection,+)from products.data_modeling.backend.graph import Graph+from products.data_modeling.backend.logic.incremental import TRINO_INCREMENTAL_SCOPE, clear_incremental_statefrom products.data_modeling.backend.logic.node_suspension import unsuspend_nodesfrom products.data_modeling.backend.logic.saved_query_dag_sync import MissingDagNodeError-from products.data_modeling.backend.models import Node+from products.data_modeling.backend.models import DataModelingJobEngine, Nodefrom products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQueryfrom products.data_modeling.backend.schedule import get_v2_saved_query_ids@@ -29,6 +45,12 @@ logger = structlog.get_logger(__name__)RELATED_RUN_EDIT_ACCESS_REFUSAL = "You need edit access to every view in this run."+MANAGED_WAREHOUSE_UNAVAILABLE_REFUSAL = "This organization has no managed warehouse set up."+# Long enough for the state changes that follow the start to commit before the workflow first reads them.+MANUAL_TRINO_RUN_START_DELAY = timedelta(seconds=5)+TRINO_RUN_ALREADY_RUNNING_REFUSAL = (+ "A managed warehouse run is already in progress for this project's models. Try again once it finishes."+)def start_node_materialization(node: Node, *, resume: bool = True, triggered_by_id: int | None = None) -> None:@@ -66,18 +88,35 @@ def start_node_materialization(node: Node, *, resume: bool = True, triggered_by_)-def run_related_node_materializations(- team_id: int,- node_id: UUID | str,- *,- direction: MaterializationRunDirection | str,- user: User,- hidden_types: frozenset[str] = frozenset(),-) -> list[str]:+def _get_node(team_id: int, node_id: UUID | str) -> Node:try:- node = Node.objects.select_related("team").get(team_id=team_id, id=node_id)+ return Node.objects.select_related("team").get(team_id=team_id, id=node_id)except Node.DoesNotExist:raise MissingDagNodeError(f"Node {node_id} not found")+++def _assert_user_can_run_nodes(node: Node, node_ids: set[str], user: User) -> None:+ # A run materializes every node it touches, so it needs the same edit and data access checks as+ # enabling materialization for each view.+ database = Database.create_for(team_id=node.team_id, user=user)+ access_control = UserAccessControl(user=user, team=node.team)+ saved_queries = DataWarehouseSavedQuery.objects.filter(+ team_id=node.team_id,+ node__id__in=node_ids,+ ).distinct()+ for saved_query in saved_queries:+ if not access_control.check_access_level_for_object(saved_query, "editor"):+ raise MaterializationForbiddenError(RELATED_RUN_EDIT_ACCESS_REFUSAL)+ assert_user_can_read_query(saved_query.query, node.team_id, user, database=database)+++def _related_node_ids(+ node: Node,+ *,+ direction: MaterializationRunDirection | str,+ user: User,+ hidden_types: frozenset[str],+) -> set[str]:graph = Graph(team_id=node.team_id, dag_id=node.dag_id, hidden_types=hidden_types)if direction == MaterializationRunDirection.UPSTREAM:node_ids = graph.get_upstream(node.id)@@ -95,18 +134,20 @@ def run_related_node_materializations(node_ids = managed_view_node_ids if node.belongs_to_managed_viewset else node_ids - managed_view_node_idsnode_ids.add(str(node.id))- # A dependency run materializes every node it touches, so it needs the same edit and data access checks as- # enabling materialization for each view.- database = Database.create_for(team_id=node.team_id, user=user)- access_control = UserAccessControl(user=user, team=node.team)- saved_queries = DataWarehouseSavedQuery.objects.filter(- team_id=node.team_id,- node__id__in=node_ids,- ).distinct()- for saved_query in saved_queries:- if not access_control.check_access_level_for_object(saved_query, "editor"):- raise MaterializationForbiddenError(RELATED_RUN_EDIT_ACCESS_REFUSAL)- assert_user_can_read_query(saved_query.query, node.team_id, user, database=database)+ _assert_user_can_run_nodes(node, node_ids, user)+ return node_ids+++def run_related_node_materializations(+ team_id: int,+ node_id: UUID | str,+ *,+ direction: MaterializationRunDirection | str,+ user: User,+ hidden_types: frozenset[str] = frozenset(),+) -> list[str]:+ node = _get_node(team_id, node_id)+ node_ids = _related_node_ids(node, direction=direction, user=user, hidden_types=hidden_types)# The workflow skips suspended nodes, so an explicit run must resume every selected node first.unsuspend_nodes(Node.objects.filter(team_id=node.team_id, id__in=node_ids), by="manual_run")@@ -134,6 +175,86 @@ def run_related_node_materializations(return sorted(node_ids)+def run_trino_node_materializations(+ team_id: int,+ node_id: UUID | str,+ *,+ direction: MaterializationRunDirection | str | None,+ user: User,+ full_refresh: bool = False,+ hidden_types: frozenset[str] = frozenset(),+) -> list[str]:+ """Build the node, and optionally its upstreams or downstreams, on the managed warehouse only.++ Runs the same managed_warehouse_only DAG run as the scheduled Trino shadow, so ClickHouse is not+ touched. Refuses while any Trino run of the DAG is in progress, because two runs would write the+ same Trino tables at once.++ `full_refresh` drops the node's Trino watermark, and only after every refusal, including losing the+ race for the run slot, so a refused run does not turn the next scheduled shadow into a full rebuild.+ The run starts a few seconds late, so the state changes commit before the workflow reads them.+ """+ node = _get_node(team_id, node_id)+ if not is_managed_warehouse_shadow_enabled(node.team):+ raise ManagedWarehouseUnavailableError(MANAGED_WAREHOUSE_UNAVAILABLE_REFUSAL)+ if direction:+ node_ids = _related_node_ids(node, direction=direction, user=user, hidden_types=hidden_types)+ else:+ node_ids = {str(node.id)}+ # The managed warehouse reads source tables without the caller's access control, so check it here.+ _assert_user_can_run_nodes(node, node_ids, user)++ dag_id = str(node.dag_id)+ temporal = sync_connect()+ inputs = ExecuteDAGInputs(+ team_id=node.team_id,+ dag_id=dag_id,+ node_ids=sorted(node_ids),+ managed_warehouse_only=True,+ shadow=True,+ manually_triggered_by_id=user.pk,+ )+ # The scheduled shadow guards overlap only per tier, by its fixed id, so ask Temporal about every tier.+ running = asyncio.run(temporal.count_workflows(query=trino_runs_of_dag_query(dag_id)))+ if running.count > 0:+ raise MaterializationAlreadyRunningError(TRINO_RUN_ALREADY_RUNNING_REFUSAL)++ # Take the run slot before changing any state, so a run that loses the race for it changes nothing. The+ # start is delayed so that the changes below commit before the workflow reads them.+ try:+ handle = asyncio.run(+ temporal.start_workflow(+ "data-modeling-execute-dag",+ asdict(inputs),+ # A fixed id makes Temporal refuse a second manual run that slipped past the count above.+ id=manual_trino_workflow_id(dag_id),+ task_queue=str(settings.DATA_MODELING_TASK_QUEUE),+ id_conflict_policy=WorkflowIDConflictPolicy.FAIL,+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE,+ execution_timeout=TRINO_SHADOW_RUN_TIMEOUT,+ retry_policy=RetryPolicy(maximum_attempts=1),+ start_delay=MANUAL_TRINO_RUN_START_DELAY,+ )+ )+ except WorkflowAlreadyStartedError:+ raise MaterializationAlreadyRunningError(TRINO_RUN_ALREADY_RUNNING_REFUSAL)++ try:+ with transaction.atomic():+ if full_refresh and node.saved_query is not None:+ clear_incremental_state(node.saved_query, scope=TRINO_INCREMENTAL_SCOPE)+ unsuspend_nodes(+ Node.objects.filter(team_id=node.team_id, id__in=node_ids),+ by="manual_run",+ engine=DataModelingJobEngine.MANAGED_WAREHOUSE,+ )+ except Exception:+ # The run would otherwise start on state the caller did not ask for, while the caller sees an error.+ asyncio.run(handle.terminate("Could not prepare the models for the run"))+ raise+ return sorted(node_ids)++def is_saved_query_on_v2_schedule(saved_query: DataWarehouseSavedQuery) -> bool:"""Whether the saved query's DAG already runs on a v2 schedule.diff --git a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py b/products/data_warehouse/backend/presentation/views/saved_query/viewset.pyindex e92a9f82c9e9..91752aa5b991 100644--- a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py+++ b/products/data_warehouse/backend/presentation/views/saved_query/viewset.py@@ -17,6 +17,7 @@ from posthog.hogql.database.database import Databasefrom posthog.api.mixins import ValidatedRequest, validated_requestfrom posthog.api.routing import TeamAndOrgViewSetMixin+from posthog.exceptions import Conflictfrom posthog.helpers.impersonation import is_impersonatedfrom posthog.models import Userfrom posthog.models.activity_logging.activity_log import Detail, load_activity, log_activity@@ -26,6 +27,8 @@ from posthog.temporal.common.client import sync_connectfrom products.access_control.backend.presentation.access_control import AccessControlViewSetMixinfrom products.data_modeling.backend.facade.api import (+ MaterializationAlreadyRunningError,+ MaterializationEngine,MaterializationFailedError,MaterializationForbiddenError,MaterializationRefusedError,@@ -111,6 +114,17 @@ class SavedQueryRunSerializer(serializers.Serializer):"Omit this field to run only this view. You cannot use it with full_refresh."),)+ engine = serializers.ChoiceField(+ choices=MaterializationEngine.choices,+ required=False,+ default=MaterializationEngine.CLICKHOUSE,+ help_text=(+ "Where to build the views. 'clickhouse' (the default) refreshes the tables that queries read. "+ "'trino' builds the same views on the organization's managed warehouse only, and leaves the "+ "ClickHouse tables as they are. 'trino' returns 403 when the organization has no managed "+ "warehouse, and 409 while another managed warehouse run of these views is in progress."+ ),+ )def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:if attrs.get("direction") and attrs["full_refresh"]:@@ -280,7 +294,16 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSereturn response.Response(status=status.HTTP_204_NO_CONTENT)- @extend_schema(request=SavedQueryRunSerializer, responses={200: SavedQueryRunResponseSerializer})+ @extend_schema(+ request=SavedQueryRunSerializer,+ responses={+ 200: SavedQueryRunResponseSerializer,+ 403: OpenApiResponse(+ description="You can't edit every view in the run, or engine is 'trino' and the organization has no managed warehouse."+ ),+ 409: OpenApiResponse(description="engine is 'trino' and a managed warehouse run is already in progress."),+ },+ )@action(methods=["POST"],detail=True,@@ -294,14 +317,16 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSeMissingDagNodeError,clear_incremental_state,materialize_saved_query,+ run_trino_node_materializations,)body = SavedQueryRunSerializer(data=request.data)body.is_valid(raise_exception=True)saved_query = self.get_object()+ on_trino = body.validated_data["engine"] == MaterializationEngine.TRINO- if body.validated_data["full_refresh"]:+ if body.validated_data["full_refresh"] and not on_trino:# Dropping the watermark is the whole mechanism: the next run finds no progress to# build on and rebuilds. Commit it before dispatch, and outside any transaction the# dispatch could roll back, or the worker reads the old watermark and runs@@ -311,7 +336,7 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetry:direction = body.validated_data.get("direction")- if direction:+ if direction or on_trino:node = (Node.objects.filter(team_id=self.team_id, saved_query_id=saved_query.id).order_by("created_at")@@ -324,13 +349,23 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSeif self.user_access_control.check_access_level_for_resource("data_catalog", required_level="viewer")else frozenset({NodeType.METRIC}))- node_ids = run_related_node_materializations(- self.team_id,- node.id,- direction=direction,- user=cast(User, request.user),- hidden_types=hidden_types,- )+ if on_trino:+ node_ids = run_trino_node_materializations(+ self.team_id,+ node.id,+ direction=direction,+ user=cast(User, request.user),+ full_refresh=body.validated_data["full_refresh"],+ hidden_types=hidden_types,+ )+ else:+ node_ids = run_related_node_materializations(+ self.team_id,+ node.id,+ direction=direction,+ user=cast(User, request.user),+ hidden_types=hidden_types,+ )else:node_id = materialize_saved_query(saved_query, triggered_by_id=request.user.pk)node_ids = [str(node_id)]@@ -340,6 +375,8 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe)except MaterializationForbiddenError as error:raise exceptions.PermissionDenied(str(error))+ except MaterializationAlreadyRunningError as error:+ raise Conflict(str(error))log_activity(organization_id=self.team.organization_id,diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.pyindex 515fd47bf682..c53b6881663d 100644--- a/products/data_warehouse/backend/tests/api/test_saved_query.py+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py@@ -18,6 +18,7 @@ from parameterized import parameterizedfrom rest_framework.parsers import JSONParserfrom rest_framework.request import Requestfrom rest_framework.test import APIRequestFactory+from temporalio.exceptions import WorkflowAlreadyStartedErrorfrom posthog.hogql.errors import ExposedHogQLError@@ -2898,6 +2899,111 @@ class TestSavedQueryRun(APIBaseTest):set(mock_client.start_workflow.call_args.args[1]["node_ids"]),{str(node.id), str(related_node.id)},)+ self.assertFalse(mock_client.start_workflow.call_args.args[1]["managed_warehouse_only"])++ @parameterized.expand([("only_this_view", None), ("with_upstream", "upstream")])+ @patch("products.data_modeling.backend.logic.node_materialization.is_managed_warehouse_shadow_enabled")+ @patch("products.data_modeling.backend.logic.node_materialization.sync_connect")+ def test_run_on_trino_starts_a_managed_warehouse_dag_run(self, _name, direction, mock_sync_connect, mock_eligible):+ saved_query, dag, node = self._make_saved_query_with_node("trino_view")+ related_saved_query = DataWarehouseSavedQuery.objects.create(+ name="trino_upstream", team=self.team, query={"query": "SELECT 1", "kind": "HogQLQuery"}+ )+ related_node = Node.objects.create(team=self.team, dag=dag, saved_query=related_saved_query, type=NodeType.VIEW)+ Edge.objects.create(team=self.team, dag=dag, source=related_node, target=node)+ mark_node_suspended(node, engine=DataModelingJobEngine.MANAGED_WAREHOUSE, reason="boom", job_id="job-1")+ node.save()+ mock_eligible.return_value = True+ mock_client = AsyncMock()+ mock_client.count_workflows.return_value = mock.Mock(count=0)+ mock_sync_connect.return_value = mock_client+ body: dict[str, Any] = {"engine": "trino"}+ if direction:+ body["direction"] = direction++ response = self.client.post(+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query.id}/run/", body+ )++ self.assertEqual(response.status_code, 200, response.content)+ expected = {str(node.id), str(related_node.id)} if direction else {str(node.id)}+ self.assertEqual(set(response.json()["node_ids"]), expected)+ mock_client.start_workflow.assert_called_once()+ call = mock_client.start_workflow.call_args+ self.assertEqual(call.args[0], "data-modeling-execute-dag")+ inputs = call.args[1]+ self.assertEqual(set(inputs["node_ids"]), expected)+ self.assertTrue(inputs["managed_warehouse_only"])+ self.assertTrue(inputs["shadow"])+ self.assertEqual(inputs["manually_triggered_by_id"], self.user.pk)+ self.assertEqual(call.kwargs["id"], f"execute-dag-trino-{dag.id}-manual")+ self.assertGreater(call.kwargs["start_delay"], timedelta(0))+ self.assertIn(f"execute-dag-trino-{dag.id}", mock_client.count_workflows.call_args.kwargs["query"])+ node.refresh_from_db()+ self.assertNotIn(DataModelingJobEngine.MANAGED_WAREHOUSE, suspension_state(node))++ @patch("products.data_modeling.backend.logic.node_materialization.is_managed_warehouse_shadow_enabled")+ @patch("products.data_modeling.backend.logic.node_materialization.sync_connect")+ def test_run_on_trino_without_a_managed_warehouse_is_forbidden(self, mock_sync_connect, mock_eligible):+ saved_query, _dag, _node = self._make_saved_query_with_node("no_warehouse_view")+ mock_eligible.return_value = False+ mock_client = AsyncMock()+ mock_sync_connect.return_value = mock_client++ response = self.client.post(+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query.id}/run/", {"engine": "trino"}+ )++ self.assertEqual(response.status_code, 403, response.content)+ mock_client.start_workflow.assert_not_called()++ @patch("products.data_modeling.backend.logic.node_materialization.is_managed_warehouse_shadow_enabled")+ @patch("products.data_modeling.backend.logic.node_materialization.sync_connect")+ def test_run_on_trino_checks_access_to_the_single_view(self, mock_sync_connect, mock_eligible):+ saved_query, _dag, _node = self._make_saved_query_with_node("restricted_view")+ mock_eligible.return_value = True+ mock_client = AsyncMock()+ mock_client.count_workflows.return_value = mock.Mock(count=0)+ mock_sync_connect.return_value = mock_client++ with patch(+ "products.data_modeling.backend.logic.node_materialization.UserAccessControl.check_access_level_for_object",+ return_value=False,+ ):+ response = self.client.post(+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query.id}/run/",+ {"engine": "trino"},+ )++ self.assertEqual(response.status_code, 403, response.content)+ mock_client.start_workflow.assert_not_called()++ @parameterized.expand([("another_run_listed", 1, False), ("start_races_a_manual_run", 0, True)])+ @patch("products.data_modeling.backend.logic.node_materialization.is_managed_warehouse_shadow_enabled")+ @patch("products.data_modeling.backend.logic.node_materialization.sync_connect")+ def test_run_on_trino_while_a_trino_run_is_in_progress_conflicts(+ self, _name, running_count, start_conflicts, mock_sync_connect, mock_eligible+ ):+ saved_query, _dag, node = self._make_saved_query_with_node("busy_view")+ mark_node_suspended(node, engine=DataModelingJobEngine.MANAGED_WAREHOUSE, reason="boom", job_id="job-1")+ node.save()+ mock_eligible.return_value = True+ mock_client = AsyncMock()+ mock_client.count_workflows.return_value = mock.Mock(count=running_count)+ if start_conflicts:+ mock_client.start_workflow.side_effect = WorkflowAlreadyStartedError("id", "data-modeling-execute-dag")+ mock_sync_connect.return_value = mock_client++ response = self.client.post(+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query.id}/run/",+ {"engine": "trino", "full_refresh": True},+ )++ self.assertEqual(response.status_code, 409, response.content)+ self.assertEqual(mock_client.start_workflow.call_count, 1 if start_conflicts else 0)+ # A refused run must leave the model as it found it, even when the refusal came after the changes.+ node.refresh_from_db()+ self.assertIn(DataModelingJobEngine.MANAGED_WAREHOUSE, suspension_state(node))@patch("products.data_modeling.backend.logic.node_materialization.sync_connect")def test_run_with_dependencies_requires_edit_access_to_every_view(self, mock_sync_connect):diff --git a/products/data_warehouse/frontend/generated/api.schemas.ts b/products/data_warehouse/frontend/generated/api.schemas.tsindex c1b726ce72fe..cfbf8527f94b 100644--- a/products/data_warehouse/frontend/generated/api.schemas.ts+++ b/products/data_warehouse/frontend/generated/api.schemas.ts@@ -2188,6 +2188,18 @@ export const MaterializationRunDirectionEnumApi = {Downstream: 'downstream',} as const+/**+ * * `clickhouse` - ClickHouse+ * * `trino` - Trino+ */+export type MaterializationEngineEnumApi =+ (typeof MaterializationEngineEnumApi)[keyof typeof MaterializationEngineEnumApi]++export const MaterializationEngineEnumApi = {+ Clickhouse: 'clickhouse',+ Trino: 'trino',+} as const+/*** Body of the `run` action.*/@@ -2199,6 +2211,11 @@ export interface SavedQueryRunApi {* * `upstream` - Upstream* * `downstream` - Downstream */direction?: MaterializationRunDirectionEnumApi+ /** Where to build the views. 'clickhouse' (the default) refreshes the tables that queries read. 'trino' builds the same views on the organization's managed warehouse only, and leaves the ClickHouse tables as they are. 'trino' returns 403 when the organization has no managed warehouse, and 409 while another managed warehouse run of these views is in progress.+ *+ * * `clickhouse` - ClickHouse+ * * `trino` - Trino */+ engine?: MaterializationEngineEnumApi}export interface SavedQueryRunResponseApi {diff --git a/products/data_warehouse/frontend/generated/api.zod.ts b/products/data_warehouse/frontend/generated/api.zod.tsindex d743a0ab940b..7d2c59f83e8d 100644--- a/products/data_warehouse/frontend/generated/api.zod.ts+++ b/products/data_warehouse/frontend/generated/api.zod.ts@@ -1016,6 +1016,7 @@ export const WarehouseSavedQueriesRevertMaterializationCreateBody = /* @__PURE__* Run this saved query.*/export const warehouseSavedQueriesRunCreateBodyFullRefreshDefault = false+export const warehouseSavedQueriesRunCreateBodyEngineDefault = `clickhouse`export const WarehouseSavedQueriesRunCreateBody = /* @__PURE__ */ zod.object({@@ -1032,6 +1033,13 @@ export const WarehouseSavedQueriesRunCreateBody = /* @__PURE__ */ zod.describe('Run this view with all upstream dependencies or all downstream dependents. Omit this field to run only this view. You cannot use it with full_refresh.\n\n\* `upstream` - Upstream\n\* `downstream` - Downstream'),+ engine: zod+ .enum(['clickhouse', 'trino'])+ .describe('\* `clickhouse` - ClickHouse\n\* `trino` - Trino')+ .default(warehouseSavedQueriesRunCreateBodyEngineDefault)+ .describe(+ "Where to build the views. 'clickhouse' (the default) refreshes the tables that queries read. 'trino' builds the same views on the organization's managed warehouse only, and leaves the ClickHouse tables as they are. 'trino' returns 403 when the organization has no managed warehouse, and 409 while another managed warehouse run of these views is in progress.\n\n\* `clickhouse` - ClickHouse\n\* `trino` - Trino"+ ),}).describe('Body of the `run` action.')diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 50fe6d5a5ab5..cf1b01c6c49d 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -71663,6 +71663,18 @@ export namespace Schemas {Value: 'value',} as const;+ /**+ * * `clickhouse` - ClickHouse+ * * `trino` - Trino+ */+ export type MaterializationEngineEnum = typeof MaterializationEngineEnum[keyof typeof MaterializationEngineEnum];+++ export const MaterializationEngineEnum = {+ Clickhouse: 'clickhouse',+ Trino: 'trino',+ } as const;+export interface MaterializationPreviewAggregate {/** Aggregate expression in the transformed query. */expression: string;@@ -106908,6 +106920,11 @@ export namespace Schemas {* * `upstream` - Upstream* * `downstream` - Downstream */direction?: MaterializationRunDirectionEnum;+ /** Where to build the views. 'clickhouse' (the default) refreshes the tables that queries read. 'trino' builds the same views on the organization's managed warehouse only, and leaves the ClickHouse tables as they are. 'trino' returns 403 when the organization has no managed warehouse, and 409 while another managed warehouse run of these views is in progress.+ *+ * * `clickhouse` - ClickHouse+ * * `trino` - Trino */+ engine?: MaterializationEngineEnum;}export interface SavedQueryRunResponse {diff --git a/services/mcp/src/generated/data_warehouse/api.ts b/services/mcp/src/generated/data_warehouse/api.tsindex 41919070b8e3..c1a9bf84a574 100644--- a/services/mcp/src/generated/data_warehouse/api.ts+++ b/services/mcp/src/generated/data_warehouse/api.ts@@ -744,6 +744,7 @@ export const WarehouseSavedQueriesRunCreateParams = () => zod.object({})export const warehouseSavedQueriesRunCreateBodyFullRefreshDefault = false+export const warehouseSavedQueriesRunCreateBodyEngineDefault = `clickhouse`export const WarehouseSavedQueriesRunCreateBody = () => zod.object({@@ -760,6 +761,13 @@ export const WarehouseSavedQueriesRunCreateBody = () => zod.describe('Run this view with all upstream dependencies or all downstream dependents. Omit this field to run only this view. You cannot use it with full_refresh.\n\n\* `upstream` - Upstream\n\* `downstream` - Downstream'),+ engine: zod+ .enum(['clickhouse', 'trino'])+ .describe('\* `clickhouse` - ClickHouse\n\* `trino` - Trino')+ .default(warehouseSavedQueriesRunCreateBodyEngineDefault)+ .describe(+ "Where to build the views. 'clickhouse' (the default) refreshes the tables that queries read. 'trino' builds the same views on the organization's managed warehouse only, and leaves the ClickHouse tables as they are. 'trino' returns 403 when the organization has no managed warehouse, and 409 while another managed warehouse run of these views is in progress.\n\n\* `clickhouse` - ClickHouse\n\* `trino` - Trino"+ ),}).describe('Body of the `run` action.')diff --git a/services/mcp/src/tools/generated/data_warehouse.ts b/services/mcp/src/tools/generated/data_warehouse.tsindex 756592311361..6365d236d25e 100644--- a/services/mcp/src/tools/generated/data_warehouse.ts+++ b/services/mcp/src/tools/generated/data_warehouse.ts@@ -477,6 +477,9 @@ const viewRun = (): ToolBase<ReturnType<typeof ViewRunSchema>, WithPostHogUrl<Scif (params.direction !== undefined) {body['direction'] = params.direction}+ if (params.engine !== undefined) {+ body['engine'] = params.engine+ }const result = await context.api.request<Schemas.SavedQueryRunResponse>({method: 'POST',path: `/api/projects/${encodeURIComponent(String(projectId))}/warehouse_saved_queries/${encodeURIComponent(String(params.id))}/run/`,diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/view-run.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/view-run.jsonindex 4699266a41d2..4eee5d9d5050 100644--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/view-run.json+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/view-run.json@@ -6,6 +6,12 @@"enum": ["upstream", "downstream"],"type": "string"},+ "engine": {+ "default": "clickhouse",+ "description": "Where to build the views. 'clickhouse' (the default) refreshes the tables that queries read. 'trino' builds the same views on the organization's managed warehouse only, and leaves the ClickHouse tables as they are. 'trino' returns 403 when the organization has no managed warehouse, and 409 while another managed warehouse run of these views is in progress.\n\n* `clickhouse` - ClickHouse\n* `trino` - Trino",+ "enum": ["clickhouse", "trino"],+ "type": "string"+ },"full_refresh": {"default": false,"description": "Rebuild the whole table instead of updating it incrementally. Has no effect on a view that is not incremental. This is how you reprocess history after changing what the query means without changing its text, or after upstream data was corrected.",