:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): refuse manual view runs within 15 minutes of the last run (#128442)
Vasco de Krijger0c30b96356b2
diff --git a/frontend/src/scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic.tsx b/frontend/src/scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic.tsxindex 6afe20f8160e..9369faee08bb 100644--- a/frontend/src/scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic.tsx+++ b/frontend/src/scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic.tsx@@ -847,8 +847,14 @@ export const dataWarehouseViewsLogic = kea<dataWarehouseViewsLogicType>([actions.runDataWarehouseSavedQuerySuccess(viewId)lemonToast.success(fullRefresh ? 'Full refresh started' : 'Materialization started')actions.loadDataWarehouseSavedQueries()- } catch {- lemonToast.error(fullRefresh ? 'Failed to start full refresh' : 'Failed to run materialization')+ } catch (error: any) {+ const retryAfter = error?.formattedRetryAfter+ lemonToast.error(+ retryAfter+ ? `This view can't run again yet. Try again ${retryAfter}.`+ : error?.detail ||+ (fullRefresh ? 'Failed to start full refresh' : 'Failed to run materialization')+ )actions.runDataWarehouseSavedQueryFailure(viewId)}},diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.pyindex be64bab3d047..4046087cfa03 100644--- a/products/data_modeling/backend/facade/api.py+++ b/products/data_modeling/backend/facade/api.py@@ -38,6 +38,7 @@ _LAZY = {"SavedQueryNotOnV2ScheduleError": "logic.node_materialization","is_saved_query_on_v2_schedule": "logic.node_materialization","materialize_saved_query": "logic.node_materialization",+ "raise_if_ran_recently": "logic.node_materialization","run_related_node_materializations": "logic.node_materialization","run_trino_node_materializations": "logic.node_materialization","run_saved_query_materialization": "logic.node_materialization",@@ -69,6 +70,7 @@ _LAZY = {"MaterializationFailedError": "facade.contracts","MaterializationForbiddenError": "facade.contracts","MaterializationAlreadyRunningError": "facade.contracts",+ "MaterializationRanRecentlyError": "facade.contracts","MaterializationEngine": "facade.contracts","ManagedWarehouseUnavailableError": "facade.contracts","MaterializationRunDirection": "facade.contracts",diff --git a/products/data_modeling/backend/facade/contracts.py b/products/data_modeling/backend/facade/contracts.pyindex 1795f3f223eb..16a53692e7d7 100644--- a/products/data_modeling/backend/facade/contracts.py+++ b/products/data_modeling/backend/facade/contracts.py@@ -133,5 +133,11 @@ class MaterializationAlreadyRunningError(Exception):pass+class MaterializationRanRecentlyError(Exception):+ def __init__(self, message: str, *, retry_after_seconds: int) -> None:+ super().__init__(message)+ self.retry_after_seconds = retry_after_seconds++class MaterializationFailedError(Exception):passdiff --git a/products/data_modeling/backend/logic/node_materialization.py b/products/data_modeling/backend/logic/node_materialization.pyindex 9ffa73768179..8c1ec3e8847f 100644--- a/products/data_modeling/backend/logic/node_materialization.py+++ b/products/data_modeling/backend/logic/node_materialization.py@@ -1,10 +1,13 @@+import mathimport asyncio+from collections.abc import Set as AbstractSetfrom dataclasses import asdict-from datetime import timedelta+from datetime import datetime, timedeltafrom uuid import UUID, uuid4from django.conf import settingsfrom django.db import transaction+from django.utils import timezoneimport structlogfrom temporalio.client import Client, WorkflowExecutionStatus@@ -32,14 +35,23 @@ from products.data_modeling.backend.facade.contracts import (ManagedWarehouseUnavailableError,MaterializationAlreadyRunningError,MaterializationForbiddenError,+ MaterializationRanRecentlyError,MaterializationRunDirection,)from products.data_modeling.backend.graph import Graph+from products.data_modeling.backend.logic.freshness import SCHEDULABLE_BUCKETS, humanize_cadencefrom products.data_modeling.backend.logic.incremental import TRINO_INCREMENTAL_SCOPE, clear_incremental_statefrom products.data_modeling.backend.logic.node_suspension import unsuspend_nodesfrom products.data_modeling.backend.logic.saved_query_dag_sync import MissingDagNodeError-from products.data_modeling.backend.models import DataModelingJobEngine, Node+from products.data_modeling.backend.models import (+ DataModelingJob,+ DataModelingJobEngine,+ DataModelingJobStatus,+ Node,+ NodeType,+)from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery+from products.data_modeling.backend.models.node import SAVED_QUERY_NODE_TYPESfrom products.data_modeling.backend.schedule import get_v2_saved_query_idslogger = structlog.get_logger(__name__)@@ -47,6 +59,18 @@ logger = structlog.get_logger(__name__)RELATED_RUN_EDIT_ACCESS_REFUSAL = "You need edit access to every view in this run."MANAGED_WAREHOUSE_UNAVAILABLE_REFUSAL = "This organization has no managed warehouse set up."+# A manual run gets the same floor as a schedule, so a script calling run in a loop cannot rebuild a view+# more often than the shortest schedule a person may pick.+MANUAL_RUN_MIN_INTERVAL = min(SCHEDULABLE_BUCKETS)+RAN_RECENTLY_REFUSAL = (+ f"This view ran less than {humanize_cadence(MANUAL_RUN_MIN_INTERVAL)} ago, so it can't run again yet."+)+RELATED_RUN_RAN_RECENTLY_REFUSAL = (+ f"Every view in this run ran less than {humanize_cadence(MANUAL_RUN_MIN_INTERVAL)} ago, so it can't run again yet. "+ "To retry a failed view now, run that view on its own."+)+# A DAG run treats plain views as ephemeral and records no job for them, so they cannot show a recent run.+DAG_BUILT_NODE_TYPES = SAVED_QUERY_NODE_TYPES - {NodeType.VIEW}# Long enough for the state changes that follow the start to commit before the workflow first reads them.MANUAL_TRINO_RUN_START_DELAY = timedelta(seconds=5)MAX_NAMED_BUILDING_MODELS = 5@@ -89,6 +113,67 @@ def start_node_materialization(node: Node, *, resume: bool = True, triggered_by_)+def _latest_runs_inside_limit(+ team_id: int,+ saved_query_ids: AbstractSet[UUID | str],+ *,+ now: datetime,+ ignored_statuses: frozenset[str] = frozenset(),+) -> list[tuple[str, datetime]]:+ """The status and start of the latest ClickHouse run of each view inside MANUAL_RUN_MIN_INTERVAL.++ A query edit after that run started removes the view, because that run did not build the new query.+ """+ latest_runs = (+ DataModelingJob.objects.filter(+ team_id=team_id,+ saved_query_id__in=saved_query_ids,+ engine=DataModelingJobEngine.CLICKHOUSE,+ created_at__gt=now - MANUAL_RUN_MIN_INTERVAL,+ )+ .exclude(status__in=ignored_statuses)+ .order_by("saved_query_id", "-created_at")+ .distinct("saved_query_id")+ .values_list("status", "created_at", "saved_query__query_updated_at")+ )+ return [+ (status, started_at)+ for status, started_at, query_updated_at in latest_runs+ if query_updated_at is None or query_updated_at <= started_at+ ]+++def _seconds_until_limit_ends(started_at: datetime, now: datetime) -> int:+ return math.ceil((started_at + MANUAL_RUN_MIN_INTERVAL - now).total_seconds())+++def raise_if_ran_recently(team_id: int, saved_query_id: UUID | str) -> None:+ """Refuse a manual run of one view inside the limit. A failed or cancelled latest run can retry at once."""+ now = timezone.now()+ for status, started_at in _latest_runs_inside_limit(+ team_id, {saved_query_id}, now=now, ignored_statuses=frozenset({DataModelingJobStatus.SKIPPED})+ ):+ if status in (DataModelingJobStatus.RUNNING, DataModelingJobStatus.COMPLETED):+ raise MaterializationRanRecentlyError(+ RAN_RECENTLY_REFUSAL, retry_after_seconds=_seconds_until_limit_ends(started_at, now)+ )+++def _raise_if_every_view_ran_recently(team_id: int, saved_query_ids: AbstractSet[UUID | str]) -> None:+ """Refuse a manual run when every view it would build had a run inside the limit, whatever that run's status.++ A failed or skipped view counts here, or one broken view would lift the limit for every view it runs with.+ The wait is for the view that leaves the limit first, because a retry can succeed from that moment.+ """+ now = timezone.now()+ latest_runs = _latest_runs_inside_limit(team_id, saved_query_ids, now=now)+ if latest_runs and len(latest_runs) == len(saved_query_ids):+ raise MaterializationRanRecentlyError(+ RELATED_RUN_RAN_RECENTLY_REFUSAL,+ retry_after_seconds=min(_seconds_until_limit_ends(started_at, now) for _, started_at in latest_runs),+ )++def _get_node(team_id: int, node_id: UUID | str) -> Node:try:return Node.objects.select_related("team").get(team_id=team_id, id=node_id)@@ -149,6 +234,12 @@ def run_related_node_materializations() -> list[str]:node = _get_node(team_id, node_id)node_ids = _related_node_ids(node, direction=direction, user=user, hidden_types=hidden_types)+ built_saved_query_ids = set(+ Node.objects.filter(team_id=team_id, id__in=node_ids, type__in=DAG_BUILT_NODE_TYPES)+ .exclude(saved_query__deleted=True)+ .values_list("saved_query_id", flat=True)+ )+ _raise_if_every_view_ran_recently(team_id, built_saved_query_ids)# The workflow skips suspended nodes, so an explicit run must resume every selected node first.unsuspend_nodes(Node.objects.filter(team_id=node.team_id, id__in=node_ids), by="manual_run")diff --git a/products/data_modeling/backend/presentation/views/node.py b/products/data_modeling/backend/presentation/views/node.pyindex 6d9c1c350a62..9e8ba2c41aa9 100644--- a/products/data_modeling/backend/presentation/views/node.py+++ b/products/data_modeling/backend/presentation/views/node.py@@ -6,10 +6,10 @@ from django.db import models, transactionfrom django.db.models import Exists, OuterRef, Subqueryfrom drf_spectacular.types import OpenApiTypes-from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_field+from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema, extend_schema_fieldfrom rest_framework import filters, mixins, request, response, serializers, status, viewsetsfrom rest_framework.decorators import action-from rest_framework.exceptions import NotFound, PermissionDenied+from rest_framework.exceptions import NotFound, PermissionDenied, Throttledfrom posthog.api.pagination import BoundedPageNumberPaginationfrom posthog.api.routing import TeamAndOrgViewSetMixin@@ -21,6 +21,7 @@ from posthog.rbac.query_access import assert_user_can_read_queryfrom products.data_modeling.backend.facade.api import (SCHEDULABLE_BUCKETS,MaterializationForbiddenError,+ MaterializationRanRecentlyError,MaterializationRunDirection,SavedQueryFrequencyBounds,UnsatisfiableFrequencyError,@@ -678,7 +679,15 @@ class NodeViewSet(self.ordering)- @extend_schema(request=NodeRunSerializer, responses={200: NodeRunResponseSerializer})+ @extend_schema(+ request=NodeRunSerializer,+ responses={+ 200: NodeRunResponseSerializer,+ 429: OpenApiResponse(+ description="Every view this run would build ran too recently and has not changed since. The Retry-After header gives the wait in seconds."+ ),+ },+ )@action(methods=["POST"], detail=True)def run(self, req: request.Request, *args, **kwargs) -> response.Response:"""Run this node with its upstream dependencies or downstream dependents."""@@ -707,6 +716,8 @@ class NodeViewSet()except MaterializationForbiddenError as error:raise PermissionDenied(str(error))+ except MaterializationRanRecentlyError as error:+ raise Throttled(wait=error.retry_after_seconds, detail=str(error))return response.Response({"node_ids": node_ids}, status=status.HTTP_200_OK)def _lineage_lookup(self, req: request.Request) -> dict[str, UUID]:@@ -784,10 +795,20 @@ class NodeViewSet(LineageResponseSerializer({"nodes": nodes, "edges": edges}, context=self.get_serializer_context()).data)+ @extend_schema(+ request=None,+ responses={+ 200: OpenApiResponse(description="Materialization started. The response has no body."),+ 400: OpenApiResponse(description="The node is not a view, materialized view or endpoint node."),+ 429: OpenApiResponse(+ description="The view's latest run started too recently and the query has not changed since. The Retry-After header gives the wait in seconds."+ ),+ },+ )@action(methods=["POST"], detail=True)def materialize(self, req: request.Request, *args, **kwargs) -> response.Response:"""Materialize just this single node."""- from products.data_modeling.backend.facade.api import start_node_materialization+ from products.data_modeling.backend.facade.api import raise_if_ran_recently, start_node_materializationnode = self.get_object()@@ -803,6 +824,10 @@ class NodeViewSet(self._require_view_edit_access(node.saved_query)if node.saved_query is not None:assert_user_can_read_query(node.saved_query.query, self.team_id, cast(User, req.user))+ try:+ raise_if_ran_recently(self.team_id, node.saved_query.id)+ except MaterializationRanRecentlyError as error:+ raise Throttled(wait=error.retry_after_seconds, detail=str(error))start_node_materialization(node, triggered_by_id=req.user.pk)diff --git a/products/data_modeling/backend/tests/api/test_node_api.py b/products/data_modeling/backend/tests/api/test_node_api.pyindex 0cf3a7d9d0ba..2ec4f8720e6b 100644--- a/products/data_modeling/backend/tests/api/test_node_api.py+++ b/products/data_modeling/backend/tests/api/test_node_api.py@@ -516,6 +516,29 @@ class TestNodeViewSet(APIBaseTest):self.assertEqual(call_args.kwargs["id_reuse_policy"], WorkflowIDReusePolicy.ALLOW_DUPLICATE)self.assertEqual(call_args.kwargs["retry_policy"].maximum_attempts, 1)+ @parameterized.expand([("materialize", {}), ("run", {"direction": "downstream"})])+ @patch("products.data_modeling.backend.logic.node_materialization.sync_connect")+ def test_a_manual_run_is_refused_soon_after_the_last_run(self, action, body, mock_sync_connect):+ mock_client = AsyncMock()+ mock_sync_connect.return_value = mock_client+ self.view_node.type = NodeType.MAT_VIEW+ self.view_node.save(update_fields=["type"])+ job = DataModelingJob.objects.create(+ team=self.team,+ saved_query=self.view_node.saved_query,+ status=DataModelingJob.Status.COMPLETED,+ engine=DataModelingJobEngine.CLICKHOUSE,+ )+ DataModelingJob.objects.filter(id=job.id).update(created_at=timezone.now() - timedelta(minutes=5))++ response = self.client.post(+ f"/api/environments/{self.team.id}/data_modeling_nodes/{self.view_node.id}/{action}/",+ body,+ )++ self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS, response.content)+ mock_client.start_workflow.assert_not_called()+def test_lineage_returns_subgraph(self):response = self.client.get(f"/api/environments/{self.team.id}/data_modeling_nodes/lineage/?node_id={self.view_node.id}"diff --git a/products/data_modeling/frontend/generated/api.ts b/products/data_modeling/frontend/generated/api.tsindex 0337794240e7..88f8e0bb7688 100644--- a/products/data_modeling/frontend/generated/api.ts+++ b/products/data_modeling/frontend/generated/api.ts@@ -268,14 +268,11 @@ export const getDataModelingNodesMaterializeCreateUrl = (projectId: string, id:export const dataModelingNodesMaterializeCreate = async (projectId: string,id: string,- nodeApi: NonReadonly<NodeApi>,options?: RequestInit-): Promise<NodeApi> => {- return apiMutator<NodeApi>(getDataModelingNodesMaterializeCreateUrl(projectId, id), {+): Promise<void> => {+ return apiMutator<void>(getDataModelingNodesMaterializeCreateUrl(projectId, id), {...options,method: 'POST',- headers: { 'Content-Type': 'application/json', ...options?.headers },- body: JSON.stringify(nodeApi),})}diff --git a/products/data_modeling/frontend/generated/api.zod.ts b/products/data_modeling/frontend/generated/api.zod.tsindex 5590100d40a9..9a8adf7d9d30 100644--- a/products/data_modeling/frontend/generated/api.zod.ts+++ b/products/data_modeling/frontend/generated/api.zod.ts@@ -77,39 +77,6 @@ export const DataModelingNodesPartialUpdateBody = /* @__PURE__ */ zod.object({),})-/**- * Materialize just this single node.- */-export const dataModelingNodesMaterializeCreateBodyNameMax = 2048--export const dataModelingNodesMaterializeCreateBodyDescriptionMax = 1024--export const DataModelingNodesMaterializeCreateBody = /* @__PURE__ */ zod.object({- name: zod.string().max(dataModelingNodesMaterializeCreateBodyNameMax),- type: zod- .enum(['table', 'view', 'matview', 'endpoint', 'metric'])- .optional()- .describe(- '\* `table` - Table\n\* `view` - View\n\* `matview` - Mat View\n\* `endpoint` - Endpoint\n\* `metric` - Metric'- ),- description: zod.string().max(dataModelingNodesMaterializeCreateBodyDescriptionMax).optional(),- sync_frequency: zod- .enum(['never', '15min', '30min', '1hour', '6hour', '12hour', '24hour', '7day', '30day'])- .describe(- '\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day'- )- .optional()- .describe(- "How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.\n\n\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day"- ),- pinned: zod- .boolean()- .optional()- .describe(- 'Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again.'- ),-})-/*** Run this node with its upstream dependencies or downstream dependents.*/diff --git a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py b/products/data_warehouse/backend/presentation/views/saved_query/viewset.pyindex 521668fb6093..236db9b537b6 100644--- a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py+++ b/products/data_warehouse/backend/presentation/views/saved_query/viewset.py@@ -31,6 +31,7 @@ from products.data_modeling.backend.facade.api import (MaterializationEngine,MaterializationFailedError,MaterializationForbiddenError,+ MaterializationRanRecentlyError,MaterializationRefusedError,MaterializationRunDirection,SavedQueryNotFoundError,@@ -318,6 +319,9 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSedescription="You can't edit every view in the run, or engine is 'trino' and the organization has no managed warehouse."),409: OpenApiResponse(description="engine is 'trino' and a managed warehouse run is already in progress."),+ 429: OpenApiResponse(+ description="Every view this run would build ran too recently and has not changed since. Runs with engine 'trino' have no such limit. The Retry-After header gives the wait in seconds."+ ),},)@action(@@ -333,6 +337,7 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSeMissingDagNodeError,clear_incremental_state,materialize_saved_query,+ raise_if_ran_recently,run_trino_node_materializations,)@@ -343,17 +348,9 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSeif saved_query.is_raw_managed_trino:raise exceptions.ValidationError(detail="Views that run raw Trino SQL cannot be materialized yet.")on_trino = body.validated_data["engine"] == MaterializationEngine.TRINO-- if body.validated_data["full_refresh"] and not on_trino:- # Dropping the watermark is the whole mechanism: the next run finds no progress to- # build on and rebuilds. Commit it before dispatch, and outside any transaction the- # dispatch could roll back, or the worker reads the old watermark and runs- # incrementally instead.- clear_incremental_state(saved_query)- clear_incremental_state(saved_query, scope=TRINO_INCREMENTAL_SCOPE)+ direction = body.validated_data.get("direction")try:- direction = body.validated_data.get("direction")if direction or on_trino:node = (Node.objects.filter(team_id=self.team_id, saved_query_id=saved_query.id)@@ -385,6 +382,15 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSehidden_types=hidden_types,)else:+ # Checked before the watermark is dropped below, so a refused full refresh changes nothing.+ raise_if_ran_recently(self.team_id, saved_query.id)+ if body.validated_data["full_refresh"]:+ # Dropping the watermark is the whole mechanism: the next run finds no progress to+ # build on and rebuilds. Commit it before dispatch, and outside any transaction the+ # dispatch could roll back, or the worker reads the old watermark and runs+ # incrementally instead.+ clear_incremental_state(saved_query)+ clear_incremental_state(saved_query, scope=TRINO_INCREMENTAL_SCOPE)node_id = materialize_saved_query(saved_query, triggered_by_id=request.user.pk)node_ids = [str(node_id)]except MissingDagNodeError:@@ -395,6 +401,8 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSeraise exceptions.PermissionDenied(str(error))except MaterializationAlreadyRunningError as error:raise Conflict(str(error))+ except MaterializationRanRecentlyError as error:+ raise exceptions.Throttled(wait=error.retry_after_seconds, detail=str(error))log_activity(organization_id=self.team.organization_id,diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.pyindex b3fe6c81c366..7910a36e12ec 100644--- a/products/data_warehouse/backend/tests/api/test_saved_query.py+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py@@ -3176,6 +3176,160 @@ class TestSavedQueryRun(APIBaseTest):mock_client.start_workflow.assert_called_once()self.assertEqual(mock_client.start_workflow.call_args[0][0], "data-modeling-materialize-view")+ @parameterized.expand(+ [+ ("completed_run_5_minutes_ago", [(DataModelingJob.Status.COMPLETED, 5)], None, 429),+ ("running_run_5_minutes_ago", [(DataModelingJob.Status.RUNNING, 5)], None, 429),+ ("completed_run_20_minutes_ago", [(DataModelingJob.Status.COMPLETED, 20)], None, 200),+ ("failed_run_5_minutes_ago", [(DataModelingJob.Status.FAILED, 5)], None, 200),+ ("cancelled_run_5_minutes_ago", [(DataModelingJob.Status.CANCELLED, 5)], None, 200),+ ("skipped_run_5_minutes_ago", [(DataModelingJob.Status.SKIPPED, 5)], None, 200),+ (+ "skipped_run_after_a_completed_run",+ [(DataModelingJob.Status.SKIPPED, 5), (DataModelingJob.Status.COMPLETED, 10)],+ None,+ 429,+ ),+ (+ "managed_warehouse_run_5_minutes_ago",+ [(DataModelingJob.Status.COMPLETED, 5, DataModelingJobEngine.MANAGED_WAREHOUSE)],+ None,+ 200,+ ),+ (+ "failed_run_after_a_completed_run",+ [(DataModelingJob.Status.COMPLETED, 10), (DataModelingJob.Status.FAILED, 5)],+ None,+ 200,+ ),+ ("query_edited_after_the_run", [(DataModelingJob.Status.COMPLETED, 5)], 2, 200),+ ("query_edited_while_the_run_ran", [(DataModelingJob.Status.COMPLETED, 10)], 8, 200),+ ]+ )+ @patch("products.data_modeling.backend.logic.node_materialization.sync_connect")+ def test_run_is_refused_soon_after_the_last_run(+ self, _name, runs, query_edited_minutes_ago, expected_status, mock_sync_connect+ ):+ mock_client = AsyncMock()+ mock_sync_connect.return_value = mock_client+ now = timezone.now()+ saved_query, _dag, _node = self._make_saved_query_with_node("floored_view")+ if query_edited_minutes_ago is not None:+ saved_query.query_updated_at = now - timedelta(minutes=query_edited_minutes_ago)+ saved_query.save(update_fields=["query_updated_at"])+ for job_status, started_minutes_ago, *engine in runs:+ job = DataModelingJob.objects.create(+ team=self.team,+ saved_query=saved_query,+ status=job_status,+ engine=engine[0] if engine else DataModelingJobEngine.CLICKHOUSE,+ last_run_at=now,+ )+ DataModelingJob.objects.filter(id=job.id).update(created_at=now - timedelta(minutes=started_minutes_ago))++ with time_machine.travel(now, tick=False):+ response = self.client.post(+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query.id}/run/",+ )++ self.assertEqual(response.status_code, expected_status, response.content)+ if expected_status == 429:+ self.assertEqual(response.headers["Retry-After"], str((15 - runs[-1][1]) * 60))+ mock_client.start_workflow.assert_not_called()+ else:+ mock_client.start_workflow.assert_called_once()++ @parameterized.expand(+ [+ (+ "every_built_view_ran_recently",+ 10,+ DataModelingJob.Status.COMPLETED,+ NodeType.MAT_VIEW,+ False,+ 429,+ "300",+ ),+ ("other_view_failed_recently", 3, DataModelingJob.Status.FAILED, NodeType.MAT_VIEW, False, 429, "600"),+ (+ "other_view_cancelled_recently",+ 3,+ DataModelingJob.Status.CANCELLED,+ NodeType.MAT_VIEW,+ False,+ 429,+ "600",+ ),+ ("other_view_skipped_recently", 3, DataModelingJob.Status.SKIPPED, NodeType.MAT_VIEW, False, 429, "600"),+ (+ "one_built_view_ran_before_the_limit",+ 20,+ DataModelingJob.Status.COMPLETED,+ NodeType.MAT_VIEW,+ False,+ 200,+ None,+ ),+ ("one_built_view_never_ran", None, None, NodeType.MAT_VIEW, False, 200, None),+ ("plain_view_never_ran", None, None, NodeType.VIEW, False, 429, "600"),+ ("deleted_view_never_ran", None, None, NodeType.MAT_VIEW, True, 429, "600"),+ ]+ )+ @patch("products.data_modeling.backend.logic.node_materialization.sync_connect")+ def test_run_with_direction_is_refused_only_when_every_built_view_ran_recently(+ self,+ _name,+ related_started_minutes_ago,+ related_status,+ related_type,+ related_deleted,+ expected_status,+ expected_retry_after,+ mock_sync_connect,+ ):+ mock_client = AsyncMock()+ mock_sync_connect.return_value = mock_client+ now = timezone.now()+ saved_query, dag, node = self._make_saved_query_with_node("floored_view")+ node.type = NodeType.MAT_VIEW+ mark_node_suspended(node, engine="clickhouse", reason="boom", job_id="job-1")+ node.save()+ related_saved_query = DataWarehouseSavedQuery.objects.create(+ name="floored_related",+ team=self.team,+ query={"query": "SELECT 1", "kind": "HogQLQuery"},+ deleted=related_deleted,+ )+ related_node = Node.objects.create(team=self.team, dag=dag, saved_query=related_saved_query, type=related_type)+ Edge.objects.create(team=self.team, dag=dag, source=related_node, target=node)+ runs = [(saved_query, 5, DataModelingJob.Status.COMPLETED)]+ if related_started_minutes_ago is not None:+ runs.append((related_saved_query, related_started_minutes_ago, related_status))+ for run_saved_query, started_minutes_ago, run_status in runs:+ job = DataModelingJob.objects.create(+ team=self.team,+ saved_query=run_saved_query,+ status=run_status,+ engine=DataModelingJobEngine.CLICKHOUSE,+ )+ DataModelingJob.objects.filter(id=job.id).update(created_at=now - timedelta(minutes=started_minutes_ago))++ with time_machine.travel(now, tick=False):+ response = self.client.post(+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query.id}/run/",+ {"direction": "upstream"},+ )++ self.assertEqual(response.status_code, expected_status, response.content)+ node.refresh_from_db()+ if expected_status == 429:+ self.assertEqual(response.headers["Retry-After"], expected_retry_after)+ mock_client.start_workflow.assert_not_called()+ self.assertIn("clickhouse", suspension_state(node))+ else:+ mock_client.start_workflow.assert_called_once()+ self.assertNotIn("clickhouse", suspension_state(node))+@parameterized.expand([("upstream",), ("downstream",)])@patch("products.data_modeling.backend.logic.node_materialization.sync_connect")def test_run_with_dependencies(self, direction, mock_sync_connect):@@ -3223,6 +3377,12 @@ class TestSavedQueryRun(APIBaseTest):Edge.objects.create(team=self.team, dag=dag, source=related_node, target=node)mark_node_suspended(node, engine=DataModelingJobEngine.MANAGED_WAREHOUSE, reason="boom", job_id="job-1")node.save()+ DataModelingJob.objects.create(+ team=self.team,+ saved_query=saved_query,+ status=DataModelingJob.Status.COMPLETED,+ engine=DataModelingJobEngine.CLICKHOUSE,+ )mock_eligible.return_value = Truemock_client = self._trino_client()mock_sync_connect.return_value = mock_clientdiff --git a/products/data_warehouse/mcp/tools.yaml b/products/data_warehouse/mcp/tools.yamlindex a5cd64832dc7..71584713c246 100644--- a/products/data_warehouse/mcp/tools.yaml+++ b/products/data_warehouse/mcp/tools.yaml@@ -301,7 +301,8 @@ tools:Trigger a manual materialization run for a saved query. By default, this refreshes only the selected view.Set direction to upstream to also run all dependencies. Set direction to downstream to also run alldependents. Both options include the selected view and run the views in dependency order. Do not set- full_refresh when you set direction. Use 'view-run-history' to check each view's status.+ full_refresh when you set direction. A run fails with a rate limit error when every view it would build ran+ recently, and the error says how long to wait. Use 'view-run-history' to check each view's status.view-run-history:operation: warehouse_saved_queries_run_history_retrieveenabled: truediff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex 99d8d6e589a5..4029ebbe4432 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -15249,7 +15249,7 @@}},"view-run": {- "description": "Trigger a manual materialization run for a saved query. By default, this refreshes only the selected view. Set direction to upstream to also run all dependencies. Set direction to downstream to also run all dependents. Both options include the selected view and run the views in dependency order. Do not set full_refresh when you set direction. Use 'view-run-history' to check each view's status.",+ "description": "Trigger a manual materialization run for a saved query. By default, this refreshes only the selected view. Set direction to upstream to also run all dependencies. Set direction to downstream to also run all dependents. Both options include the selected view and run the views in dependency order. Do not set full_refresh when you set direction. A run fails with a rate limit error when every view it would build ran recently, and the error says how long to wait. Use 'view-run-history' to check each view's status.","category": "Data warehouse","feature": "data_warehouse","summary": "Run saved query",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex 66c6fe898196..9f6236a1f114 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -16023,7 +16023,7 @@}},"view-run": {- "description": "Trigger a manual materialization run for a saved query. By default, this refreshes only the selected view. Set direction to upstream to also run all dependencies. Set direction to downstream to also run all dependents. Both options include the selected view and run the views in dependency order. Do not set full_refresh when you set direction. Use 'view-run-history' to check each view's status.",+ "description": "Trigger a manual materialization run for a saved query. By default, this refreshes only the selected view. Set direction to upstream to also run all dependencies. Set direction to downstream to also run all dependents. Both options include the selected view and run the views in dependency order. Do not set full_refresh when you set direction. A run fails with a rate limit error when every view it would build ran recently, and the error says how long to wait. Use 'view-run-history' to check each view's status.","category": "Data warehouse","feature": "data_warehouse","summary": "Run saved query",