Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): refuse manual view runs within 15 minutes of the last run (#128442)

Vasco de Krijger0c30b96356b2
diff --git a/frontend/src/scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic.tsx b/frontend/src/scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic.tsx
index 6afe20f8160e..9369faee08bb 100644
--- a/frontend/src/scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic.tsx
+++ b/frontend/src/scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic.tsx
@@ -847,8 +847,14 @@ export const dataWarehouseViewsLogic = kea<dataWarehouseViewsLogicType>([
actions.runDataWarehouseSavedQuerySuccess(viewId)
lemonToast.success(fullRefresh ? 'Full refresh started' : 'Materialization started')
actions.loadDataWarehouseSavedQueries()
- } catch {
- lemonToast.error(fullRefresh ? 'Failed to start full refresh' : 'Failed to run materialization')
+ } catch (error: any) {
+ const retryAfter = error?.formattedRetryAfter
+ lemonToast.error(
+ retryAfter
+ ? `This view can't run again yet. Try again ${retryAfter}.`
+ : error?.detail ||
+ (fullRefresh ? 'Failed to start full refresh' : 'Failed to run materialization')
+ )
actions.runDataWarehouseSavedQueryFailure(viewId)
}
},
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index be64bab3d047..4046087cfa03 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -38,6 +38,7 @@ _LAZY = {
"SavedQueryNotOnV2ScheduleError": "logic.node_materialization",
"is_saved_query_on_v2_schedule": "logic.node_materialization",
"materialize_saved_query": "logic.node_materialization",
+ "raise_if_ran_recently": "logic.node_materialization",
"run_related_node_materializations": "logic.node_materialization",
"run_trino_node_materializations": "logic.node_materialization",
"run_saved_query_materialization": "logic.node_materialization",
@@ -69,6 +70,7 @@ _LAZY = {
"MaterializationFailedError": "facade.contracts",
"MaterializationForbiddenError": "facade.contracts",
"MaterializationAlreadyRunningError": "facade.contracts",
+ "MaterializationRanRecentlyError": "facade.contracts",
"MaterializationEngine": "facade.contracts",
"ManagedWarehouseUnavailableError": "facade.contracts",
"MaterializationRunDirection": "facade.contracts",
diff --git a/products/data_modeling/backend/facade/contracts.py b/products/data_modeling/backend/facade/contracts.py
index 1795f3f223eb..16a53692e7d7 100644
--- a/products/data_modeling/backend/facade/contracts.py
+++ b/products/data_modeling/backend/facade/contracts.py
@@ -133,5 +133,11 @@ class MaterializationAlreadyRunningError(Exception):
pass
+class MaterializationRanRecentlyError(Exception):
+ def __init__(self, message: str, *, retry_after_seconds: int) -> None:
+ super().__init__(message)
+ self.retry_after_seconds = retry_after_seconds
+
+
class MaterializationFailedError(Exception):
pass
diff --git a/products/data_modeling/backend/logic/node_materialization.py b/products/data_modeling/backend/logic/node_materialization.py
index 9ffa73768179..8c1ec3e8847f 100644
--- a/products/data_modeling/backend/logic/node_materialization.py
+++ b/products/data_modeling/backend/logic/node_materialization.py
@@ -1,10 +1,13 @@
+import math
import asyncio
+from collections.abc import Set as AbstractSet
from dataclasses import asdict
-from datetime import timedelta
+from datetime import datetime, timedelta
from uuid import UUID, uuid4
from django.conf import settings
from django.db import transaction
+from django.utils import timezone
import structlog
from temporalio.client import Client, WorkflowExecutionStatus
@@ -32,14 +35,23 @@ from products.data_modeling.backend.facade.contracts import (
ManagedWarehouseUnavailableError,
MaterializationAlreadyRunningError,
MaterializationForbiddenError,
+ MaterializationRanRecentlyError,
MaterializationRunDirection,
)
from products.data_modeling.backend.graph import Graph
+from products.data_modeling.backend.logic.freshness import SCHEDULABLE_BUCKETS, humanize_cadence
from products.data_modeling.backend.logic.incremental import TRINO_INCREMENTAL_SCOPE, clear_incremental_state
from products.data_modeling.backend.logic.node_suspension import unsuspend_nodes
from products.data_modeling.backend.logic.saved_query_dag_sync import MissingDagNodeError
-from products.data_modeling.backend.models import DataModelingJobEngine, Node
+from products.data_modeling.backend.models import (
+ DataModelingJob,
+ DataModelingJobEngine,
+ DataModelingJobStatus,
+ Node,
+ NodeType,
+)
from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
+from products.data_modeling.backend.models.node import SAVED_QUERY_NODE_TYPES
from products.data_modeling.backend.schedule import get_v2_saved_query_ids
logger = structlog.get_logger(__name__)
@@ -47,6 +59,18 @@ logger = structlog.get_logger(__name__)
RELATED_RUN_EDIT_ACCESS_REFUSAL = "You need edit access to every view in this run."
MANAGED_WAREHOUSE_UNAVAILABLE_REFUSAL = "This organization has no managed warehouse set up."
+# A manual run gets the same floor as a schedule, so a script calling run in a loop cannot rebuild a view
+# more often than the shortest schedule a person may pick.
+MANUAL_RUN_MIN_INTERVAL = min(SCHEDULABLE_BUCKETS)
+RAN_RECENTLY_REFUSAL = (
+ f"This view ran less than {humanize_cadence(MANUAL_RUN_MIN_INTERVAL)} ago, so it can't run again yet."
+)
+RELATED_RUN_RAN_RECENTLY_REFUSAL = (
+ f"Every view in this run ran less than {humanize_cadence(MANUAL_RUN_MIN_INTERVAL)} ago, so it can't run again yet. "
+ "To retry a failed view now, run that view on its own."
+)
+# A DAG run treats plain views as ephemeral and records no job for them, so they cannot show a recent run.
+DAG_BUILT_NODE_TYPES = SAVED_QUERY_NODE_TYPES - {NodeType.VIEW}
# Long enough for the state changes that follow the start to commit before the workflow first reads them.
MANUAL_TRINO_RUN_START_DELAY = timedelta(seconds=5)
MAX_NAMED_BUILDING_MODELS = 5
@@ -89,6 +113,67 @@ def start_node_materialization(node: Node, *, resume: bool = True, triggered_by_
)
+def _latest_runs_inside_limit(
+ team_id: int,
+ saved_query_ids: AbstractSet[UUID | str],
+ *,
+ now: datetime,
+ ignored_statuses: frozenset[str] = frozenset(),
+) -> list[tuple[str, datetime]]:
+ """The status and start of the latest ClickHouse run of each view inside MANUAL_RUN_MIN_INTERVAL.
+
+ A query edit after that run started removes the view, because that run did not build the new query.
+ """
+ latest_runs = (
+ DataModelingJob.objects.filter(
+ team_id=team_id,
+ saved_query_id__in=saved_query_ids,
+ engine=DataModelingJobEngine.CLICKHOUSE,
+ created_at__gt=now - MANUAL_RUN_MIN_INTERVAL,
+ )
+ .exclude(status__in=ignored_statuses)
+ .order_by("saved_query_id", "-created_at")
+ .distinct("saved_query_id")
+ .values_list("status", "created_at", "saved_query__query_updated_at")
+ )
+ return [
+ (status, started_at)
+ for status, started_at, query_updated_at in latest_runs
+ if query_updated_at is None or query_updated_at <= started_at
+ ]
+
+
+def _seconds_until_limit_ends(started_at: datetime, now: datetime) -> int:
+ return math.ceil((started_at + MANUAL_RUN_MIN_INTERVAL - now).total_seconds())
+
+
+def raise_if_ran_recently(team_id: int, saved_query_id: UUID | str) -> None:
+ """Refuse a manual run of one view inside the limit. A failed or cancelled latest run can retry at once."""
+ now = timezone.now()
+ for status, started_at in _latest_runs_inside_limit(
+ team_id, {saved_query_id}, now=now, ignored_statuses=frozenset({DataModelingJobStatus.SKIPPED})
+ ):
+ if status in (DataModelingJobStatus.RUNNING, DataModelingJobStatus.COMPLETED):
+ raise MaterializationRanRecentlyError(
+ RAN_RECENTLY_REFUSAL, retry_after_seconds=_seconds_until_limit_ends(started_at, now)
+ )
+
+
+def _raise_if_every_view_ran_recently(team_id: int, saved_query_ids: AbstractSet[UUID | str]) -> None:
+ """Refuse a manual run when every view it would build had a run inside the limit, whatever that run's status.
+
+ A failed or skipped view counts here, or one broken view would lift the limit for every view it runs with.
+ The wait is for the view that leaves the limit first, because a retry can succeed from that moment.
+ """
+ now = timezone.now()
+ latest_runs = _latest_runs_inside_limit(team_id, saved_query_ids, now=now)
+ if latest_runs and len(latest_runs) == len(saved_query_ids):
+ raise MaterializationRanRecentlyError(
+ RELATED_RUN_RAN_RECENTLY_REFUSAL,
+ retry_after_seconds=min(_seconds_until_limit_ends(started_at, now) for _, started_at in latest_runs),
+ )
+
+
def _get_node(team_id: int, node_id: UUID | str) -> Node:
try:
return Node.objects.select_related("team").get(team_id=team_id, id=node_id)
@@ -149,6 +234,12 @@ def run_related_node_materializations(
) -> list[str]:
node = _get_node(team_id, node_id)
node_ids = _related_node_ids(node, direction=direction, user=user, hidden_types=hidden_types)
+ built_saved_query_ids = set(
+ Node.objects.filter(team_id=team_id, id__in=node_ids, type__in=DAG_BUILT_NODE_TYPES)
+ .exclude(saved_query__deleted=True)
+ .values_list("saved_query_id", flat=True)
+ )
+ _raise_if_every_view_ran_recently(team_id, built_saved_query_ids)
# The workflow skips suspended nodes, so an explicit run must resume every selected node first.
unsuspend_nodes(Node.objects.filter(team_id=node.team_id, id__in=node_ids), by="manual_run")
diff --git a/products/data_modeling/backend/presentation/views/node.py b/products/data_modeling/backend/presentation/views/node.py
index 6d9c1c350a62..9e8ba2c41aa9 100644
--- a/products/data_modeling/backend/presentation/views/node.py
+++ b/products/data_modeling/backend/presentation/views/node.py
@@ -6,10 +6,10 @@ from django.db import models, transaction
from django.db.models import Exists, OuterRef, Subquery
from drf_spectacular.types import OpenApiTypes
-from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_field
+from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema, extend_schema_field
from rest_framework import filters, mixins, request, response, serializers, status, viewsets
from rest_framework.decorators import action
-from rest_framework.exceptions import NotFound, PermissionDenied
+from rest_framework.exceptions import NotFound, PermissionDenied, Throttled
from posthog.api.pagination import BoundedPageNumberPagination
from posthog.api.routing import TeamAndOrgViewSetMixin
@@ -21,6 +21,7 @@ from posthog.rbac.query_access import assert_user_can_read_query
from products.data_modeling.backend.facade.api import (
SCHEDULABLE_BUCKETS,
MaterializationForbiddenError,
+ MaterializationRanRecentlyError,
MaterializationRunDirection,
SavedQueryFrequencyBounds,
UnsatisfiableFrequencyError,
@@ -678,7 +679,15 @@ class NodeViewSet(
self.ordering
)
- @extend_schema(request=NodeRunSerializer, responses={200: NodeRunResponseSerializer})
+ @extend_schema(
+ request=NodeRunSerializer,
+ responses={
+ 200: NodeRunResponseSerializer,
+ 429: OpenApiResponse(
+ description="Every view this run would build ran too recently and has not changed since. The Retry-After header gives the wait in seconds."
+ ),
+ },
+ )
@action(methods=["POST"], detail=True)
def run(self, req: request.Request, *args, **kwargs) -> response.Response:
"""Run this node with its upstream dependencies or downstream dependents."""
@@ -707,6 +716,8 @@ class NodeViewSet(
)
except MaterializationForbiddenError as error:
raise PermissionDenied(str(error))
+ except MaterializationRanRecentlyError as error:
+ raise Throttled(wait=error.retry_after_seconds, detail=str(error))
return response.Response({"node_ids": node_ids}, status=status.HTTP_200_OK)
def _lineage_lookup(self, req: request.Request) -> dict[str, UUID]:
@@ -784,10 +795,20 @@ class NodeViewSet(
LineageResponseSerializer({"nodes": nodes, "edges": edges}, context=self.get_serializer_context()).data
)
+ @extend_schema(
+ request=None,
+ responses={
+ 200: OpenApiResponse(description="Materialization started. The response has no body."),
+ 400: OpenApiResponse(description="The node is not a view, materialized view or endpoint node."),
+ 429: OpenApiResponse(
+ description="The view's latest run started too recently and the query has not changed since. The Retry-After header gives the wait in seconds."
+ ),
+ },
+ )
@action(methods=["POST"], detail=True)
def materialize(self, req: request.Request, *args, **kwargs) -> response.Response:
"""Materialize just this single node."""
- from products.data_modeling.backend.facade.api import start_node_materialization
+ from products.data_modeling.backend.facade.api import raise_if_ran_recently, start_node_materialization
node = self.get_object()
@@ -803,6 +824,10 @@ class NodeViewSet(
self._require_view_edit_access(node.saved_query)
if node.saved_query is not None:
assert_user_can_read_query(node.saved_query.query, self.team_id, cast(User, req.user))
+ try:
+ raise_if_ran_recently(self.team_id, node.saved_query.id)
+ except MaterializationRanRecentlyError as error:
+ raise Throttled(wait=error.retry_after_seconds, detail=str(error))
start_node_materialization(node, triggered_by_id=req.user.pk)
diff --git a/products/data_modeling/backend/tests/api/test_node_api.py b/products/data_modeling/backend/tests/api/test_node_api.py
index 0cf3a7d9d0ba..2ec4f8720e6b 100644
--- a/products/data_modeling/backend/tests/api/test_node_api.py
+++ b/products/data_modeling/backend/tests/api/test_node_api.py
@@ -516,6 +516,29 @@ class TestNodeViewSet(APIBaseTest):
self.assertEqual(call_args.kwargs["id_reuse_policy"], WorkflowIDReusePolicy.ALLOW_DUPLICATE)
self.assertEqual(call_args.kwargs["retry_policy"].maximum_attempts, 1)
+ @parameterized.expand([("materialize", {}), ("run", {"direction": "downstream"})])
+ @patch("products.data_modeling.backend.logic.node_materialization.sync_connect")
+ def test_a_manual_run_is_refused_soon_after_the_last_run(self, action, body, mock_sync_connect):
+ mock_client = AsyncMock()
+ mock_sync_connect.return_value = mock_client
+ self.view_node.type = NodeType.MAT_VIEW
+ self.view_node.save(update_fields=["type"])
+ job = DataModelingJob.objects.create(
+ team=self.team,
+ saved_query=self.view_node.saved_query,
+ status=DataModelingJob.Status.COMPLETED,
+ engine=DataModelingJobEngine.CLICKHOUSE,
+ )
+ DataModelingJob.objects.filter(id=job.id).update(created_at=timezone.now() - timedelta(minutes=5))
+
+ response = self.client.post(
+ f"/api/environments/{self.team.id}/data_modeling_nodes/{self.view_node.id}/{action}/",
+ body,
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS, response.content)
+ mock_client.start_workflow.assert_not_called()
+
def test_lineage_returns_subgraph(self):
response = self.client.get(
f"/api/environments/{self.team.id}/data_modeling_nodes/lineage/?node_id={self.view_node.id}"
diff --git a/products/data_modeling/frontend/generated/api.ts b/products/data_modeling/frontend/generated/api.ts
index 0337794240e7..88f8e0bb7688 100644
--- a/products/data_modeling/frontend/generated/api.ts
+++ b/products/data_modeling/frontend/generated/api.ts
@@ -268,14 +268,11 @@ export const getDataModelingNodesMaterializeCreateUrl = (projectId: string, id:
export const dataModelingNodesMaterializeCreate = async (
projectId: string,
id: string,
- nodeApi: NonReadonly<NodeApi>,
options?: RequestInit
-): Promise<NodeApi> => {
- return apiMutator<NodeApi>(getDataModelingNodesMaterializeCreateUrl(projectId, id), {
+): Promise<void> => {
+ return apiMutator<void>(getDataModelingNodesMaterializeCreateUrl(projectId, id), {
...options,
method: 'POST',
- headers: { 'Content-Type': 'application/json', ...options?.headers },
- body: JSON.stringify(nodeApi),
})
}
diff --git a/products/data_modeling/frontend/generated/api.zod.ts b/products/data_modeling/frontend/generated/api.zod.ts
index 5590100d40a9..9a8adf7d9d30 100644
--- a/products/data_modeling/frontend/generated/api.zod.ts
+++ b/products/data_modeling/frontend/generated/api.zod.ts
@@ -77,39 +77,6 @@ export const DataModelingNodesPartialUpdateBody = /* @__PURE__ */ zod.object({
),
})
-/**
- * Materialize just this single node.
- */
-export const dataModelingNodesMaterializeCreateBodyNameMax = 2048
-
-export const dataModelingNodesMaterializeCreateBodyDescriptionMax = 1024
-
-export const DataModelingNodesMaterializeCreateBody = /* @__PURE__ */ zod.object({
- name: zod.string().max(dataModelingNodesMaterializeCreateBodyNameMax),
- type: zod
- .enum(['table', 'view', 'matview', 'endpoint', 'metric'])
- .optional()
- .describe(
- '\* `table` - Table\n\* `view` - View\n\* `matview` - Mat View\n\* `endpoint` - Endpoint\n\* `metric` - Metric'
- ),
- description: zod.string().max(dataModelingNodesMaterializeCreateBodyDescriptionMax).optional(),
- sync_frequency: zod
- .enum(['never', '15min', '30min', '1hour', '6hour', '12hour', '24hour', '7day', '30day'])
- .describe(
- '\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day'
- )
- .optional()
- .describe(
- "How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.\n\n\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day"
- ),
- pinned: zod
- .boolean()
- .optional()
- .describe(
- 'Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again.'
- ),
-})
-
/**
* Run this node with its upstream dependencies or downstream dependents.
*/
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py b/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
index 521668fb6093..236db9b537b6 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
@@ -31,6 +31,7 @@ from products.data_modeling.backend.facade.api import (
MaterializationEngine,
MaterializationFailedError,
MaterializationForbiddenError,
+ MaterializationRanRecentlyError,
MaterializationRefusedError,
MaterializationRunDirection,
SavedQueryNotFoundError,
@@ -318,6 +319,9 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
description="You can't edit every view in the run, or engine is 'trino' and the organization has no managed warehouse."
),
409: OpenApiResponse(description="engine is 'trino' and a managed warehouse run is already in progress."),
+ 429: OpenApiResponse(
+ description="Every view this run would build ran too recently and has not changed since. Runs with engine 'trino' have no such limit. The Retry-After header gives the wait in seconds."
+ ),
},
)
@action(
@@ -333,6 +337,7 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
MissingDagNodeError,
clear_incremental_state,
materialize_saved_query,
+ raise_if_ran_recently,
run_trino_node_materializations,
)
@@ -343,17 +348,9 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
if saved_query.is_raw_managed_trino:
raise exceptions.ValidationError(detail="Views that run raw Trino SQL cannot be materialized yet.")
on_trino = body.validated_data["engine"] == MaterializationEngine.TRINO
-
- if body.validated_data["full_refresh"] and not on_trino:
- # Dropping the watermark is the whole mechanism: the next run finds no progress to
- # build on and rebuilds. Commit it before dispatch, and outside any transaction the
- # dispatch could roll back, or the worker reads the old watermark and runs
- # incrementally instead.
- clear_incremental_state(saved_query)
- clear_incremental_state(saved_query, scope=TRINO_INCREMENTAL_SCOPE)
+ direction = body.validated_data.get("direction")
try:
- direction = body.validated_data.get("direction")
if direction or on_trino:
node = (
Node.objects.filter(team_id=self.team_id, saved_query_id=saved_query.id)
@@ -385,6 +382,15 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
hidden_types=hidden_types,
)
else:
+ # Checked before the watermark is dropped below, so a refused full refresh changes nothing.
+ raise_if_ran_recently(self.team_id, saved_query.id)
+ if body.validated_data["full_refresh"]:
+ # Dropping the watermark is the whole mechanism: the next run finds no progress to
+ # build on and rebuilds. Commit it before dispatch, and outside any transaction the
+ # dispatch could roll back, or the worker reads the old watermark and runs
+ # incrementally instead.
+ clear_incremental_state(saved_query)
+ clear_incremental_state(saved_query, scope=TRINO_INCREMENTAL_SCOPE)
node_id = materialize_saved_query(saved_query, triggered_by_id=request.user.pk)
node_ids = [str(node_id)]
except MissingDagNodeError:
@@ -395,6 +401,8 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
raise exceptions.PermissionDenied(str(error))
except MaterializationAlreadyRunningError as error:
raise Conflict(str(error))
+ except MaterializationRanRecentlyError as error:
+ raise exceptions.Throttled(wait=error.retry_after_seconds, detail=str(error))
log_activity(
organization_id=self.team.organization_id,
diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.py
index b3fe6c81c366..7910a36e12ec 100644
--- a/products/data_warehouse/backend/tests/api/test_saved_query.py
+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py
@@ -3176,6 +3176,160 @@ class TestSavedQueryRun(APIBaseTest):
mock_client.start_workflow.assert_called_once()
self.assertEqual(mock_client.start_workflow.call_args[0][0], "data-modeling-materialize-view")
+ @parameterized.expand(
+ [
+ ("completed_run_5_minutes_ago", [(DataModelingJob.Status.COMPLETED, 5)], None, 429),
+ ("running_run_5_minutes_ago", [(DataModelingJob.Status.RUNNING, 5)], None, 429),
+ ("completed_run_20_minutes_ago", [(DataModelingJob.Status.COMPLETED, 20)], None, 200),
+ ("failed_run_5_minutes_ago", [(DataModelingJob.Status.FAILED, 5)], None, 200),
+ ("cancelled_run_5_minutes_ago", [(DataModelingJob.Status.CANCELLED, 5)], None, 200),
+ ("skipped_run_5_minutes_ago", [(DataModelingJob.Status.SKIPPED, 5)], None, 200),
+ (
+ "skipped_run_after_a_completed_run",
+ [(DataModelingJob.Status.SKIPPED, 5), (DataModelingJob.Status.COMPLETED, 10)],
+ None,
+ 429,
+ ),
+ (
+ "managed_warehouse_run_5_minutes_ago",
+ [(DataModelingJob.Status.COMPLETED, 5, DataModelingJobEngine.MANAGED_WAREHOUSE)],
+ None,
+ 200,
+ ),
+ (
+ "failed_run_after_a_completed_run",
+ [(DataModelingJob.Status.COMPLETED, 10), (DataModelingJob.Status.FAILED, 5)],
+ None,
+ 200,
+ ),
+ ("query_edited_after_the_run", [(DataModelingJob.Status.COMPLETED, 5)], 2, 200),
+ ("query_edited_while_the_run_ran", [(DataModelingJob.Status.COMPLETED, 10)], 8, 200),
+ ]
+ )
+ @patch("products.data_modeling.backend.logic.node_materialization.sync_connect")
+ def test_run_is_refused_soon_after_the_last_run(
+ self, _name, runs, query_edited_minutes_ago, expected_status, mock_sync_connect
+ ):
+ mock_client = AsyncMock()
+ mock_sync_connect.return_value = mock_client
+ now = timezone.now()
+ saved_query, _dag, _node = self._make_saved_query_with_node("floored_view")
+ if query_edited_minutes_ago is not None:
+ saved_query.query_updated_at = now - timedelta(minutes=query_edited_minutes_ago)
+ saved_query.save(update_fields=["query_updated_at"])
+ for job_status, started_minutes_ago, *engine in runs:
+ job = DataModelingJob.objects.create(
+ team=self.team,
+ saved_query=saved_query,
+ status=job_status,
+ engine=engine[0] if engine else DataModelingJobEngine.CLICKHOUSE,
+ last_run_at=now,
+ )
+ DataModelingJob.objects.filter(id=job.id).update(created_at=now - timedelta(minutes=started_minutes_ago))
+
+ with time_machine.travel(now, tick=False):
+ response = self.client.post(
+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query.id}/run/",
+ )
+
+ self.assertEqual(response.status_code, expected_status, response.content)
+ if expected_status == 429:
+ self.assertEqual(response.headers["Retry-After"], str((15 - runs[-1][1]) * 60))
+ mock_client.start_workflow.assert_not_called()
+ else:
+ mock_client.start_workflow.assert_called_once()
+
+ @parameterized.expand(
+ [
+ (
+ "every_built_view_ran_recently",
+ 10,
+ DataModelingJob.Status.COMPLETED,
+ NodeType.MAT_VIEW,
+ False,
+ 429,
+ "300",
+ ),
+ ("other_view_failed_recently", 3, DataModelingJob.Status.FAILED, NodeType.MAT_VIEW, False, 429, "600"),
+ (
+ "other_view_cancelled_recently",
+ 3,
+ DataModelingJob.Status.CANCELLED,
+ NodeType.MAT_VIEW,
+ False,
+ 429,
+ "600",
+ ),
+ ("other_view_skipped_recently", 3, DataModelingJob.Status.SKIPPED, NodeType.MAT_VIEW, False, 429, "600"),
+ (
+ "one_built_view_ran_before_the_limit",
+ 20,
+ DataModelingJob.Status.COMPLETED,
+ NodeType.MAT_VIEW,
+ False,
+ 200,
+ None,
+ ),
+ ("one_built_view_never_ran", None, None, NodeType.MAT_VIEW, False, 200, None),
+ ("plain_view_never_ran", None, None, NodeType.VIEW, False, 429, "600"),
+ ("deleted_view_never_ran", None, None, NodeType.MAT_VIEW, True, 429, "600"),
+ ]
+ )
+ @patch("products.data_modeling.backend.logic.node_materialization.sync_connect")
+ def test_run_with_direction_is_refused_only_when_every_built_view_ran_recently(
+ self,
+ _name,
+ related_started_minutes_ago,
+ related_status,
+ related_type,
+ related_deleted,
+ expected_status,
+ expected_retry_after,
+ mock_sync_connect,
+ ):
+ mock_client = AsyncMock()
+ mock_sync_connect.return_value = mock_client
+ now = timezone.now()
+ saved_query, dag, node = self._make_saved_query_with_node("floored_view")
+ node.type = NodeType.MAT_VIEW
+ mark_node_suspended(node, engine="clickhouse", reason="boom", job_id="job-1")
+ node.save()
+ related_saved_query = DataWarehouseSavedQuery.objects.create(
+ name="floored_related",
+ team=self.team,
+ query={"query": "SELECT 1", "kind": "HogQLQuery"},
+ deleted=related_deleted,
+ )
+ related_node = Node.objects.create(team=self.team, dag=dag, saved_query=related_saved_query, type=related_type)
+ Edge.objects.create(team=self.team, dag=dag, source=related_node, target=node)
+ runs = [(saved_query, 5, DataModelingJob.Status.COMPLETED)]
+ if related_started_minutes_ago is not None:
+ runs.append((related_saved_query, related_started_minutes_ago, related_status))
+ for run_saved_query, started_minutes_ago, run_status in runs:
+ job = DataModelingJob.objects.create(
+ team=self.team,
+ saved_query=run_saved_query,
+ status=run_status,
+ engine=DataModelingJobEngine.CLICKHOUSE,
+ )
+ DataModelingJob.objects.filter(id=job.id).update(created_at=now - timedelta(minutes=started_minutes_ago))
+
+ with time_machine.travel(now, tick=False):
+ response = self.client.post(
+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query.id}/run/",
+ {"direction": "upstream"},
+ )
+
+ self.assertEqual(response.status_code, expected_status, response.content)
+ node.refresh_from_db()
+ if expected_status == 429:
+ self.assertEqual(response.headers["Retry-After"], expected_retry_after)
+ mock_client.start_workflow.assert_not_called()
+ self.assertIn("clickhouse", suspension_state(node))
+ else:
+ mock_client.start_workflow.assert_called_once()
+ self.assertNotIn("clickhouse", suspension_state(node))
+
@parameterized.expand([("upstream",), ("downstream",)])
@patch("products.data_modeling.backend.logic.node_materialization.sync_connect")
def test_run_with_dependencies(self, direction, mock_sync_connect):
@@ -3223,6 +3377,12 @@ class TestSavedQueryRun(APIBaseTest):
Edge.objects.create(team=self.team, dag=dag, source=related_node, target=node)
mark_node_suspended(node, engine=DataModelingJobEngine.MANAGED_WAREHOUSE, reason="boom", job_id="job-1")
node.save()
+ DataModelingJob.objects.create(
+ team=self.team,
+ saved_query=saved_query,
+ status=DataModelingJob.Status.COMPLETED,
+ engine=DataModelingJobEngine.CLICKHOUSE,
+ )
mock_eligible.return_value = True
mock_client = self._trino_client()
mock_sync_connect.return_value = mock_client
diff --git a/products/data_warehouse/mcp/tools.yaml b/products/data_warehouse/mcp/tools.yaml
index a5cd64832dc7..71584713c246 100644
--- a/products/data_warehouse/mcp/tools.yaml
+++ b/products/data_warehouse/mcp/tools.yaml
@@ -301,7 +301,8 @@ tools:
Trigger a manual materialization run for a saved query. By default, this refreshes only the selected view.
Set direction to upstream to also run all dependencies. Set direction to downstream to also run all
dependents. Both options include the selected view and run the views in dependency order. Do not set
- full_refresh when you set direction. Use 'view-run-history' to check each view's status.
+ full_refresh when you set direction. A run fails with a rate limit error when every view it would build ran
+ recently, and the error says how long to wait. Use 'view-run-history' to check each view's status.
view-run-history:
operation: warehouse_saved_queries_run_history_retrieve
enabled: true
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index 99d8d6e589a5..4029ebbe4432 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -15249,7 +15249,7 @@
}
},
"view-run": {
- "description": "Trigger a manual materialization run for a saved query. By default, this refreshes only the selected view. Set direction to upstream to also run all dependencies. Set direction to downstream to also run all dependents. Both options include the selected view and run the views in dependency order. Do not set full_refresh when you set direction. Use 'view-run-history' to check each view's status.",
+ "description": "Trigger a manual materialization run for a saved query. By default, this refreshes only the selected view. Set direction to upstream to also run all dependencies. Set direction to downstream to also run all dependents. Both options include the selected view and run the views in dependency order. Do not set full_refresh when you set direction. A run fails with a rate limit error when every view it would build ran recently, and the error says how long to wait. Use 'view-run-history' to check each view's status.",
"category": "Data warehouse",
"feature": "data_warehouse",
"summary": "Run saved query",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index 66c6fe898196..9f6236a1f114 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -16023,7 +16023,7 @@
}
},
"view-run": {
- "description": "Trigger a manual materialization run for a saved query. By default, this refreshes only the selected view. Set direction to upstream to also run all dependencies. Set direction to downstream to also run all dependents. Both options include the selected view and run the views in dependency order. Do not set full_refresh when you set direction. Use 'view-run-history' to check each view's status.",
+ "description": "Trigger a manual materialization run for a saved query. By default, this refreshes only the selected view. Set direction to upstream to also run all dependencies. Set direction to downstream to also run all dependents. Both options include the selected view and run the views in dependency order. Do not set full_refresh when you set direction. A run fails with a rate limit error when every view it would build ran recently, and the error says how long to wait. Use 'view-run-history' to check each view's status.",
"category": "Data warehouse",
"feature": "data_warehouse",
"summary": "Run saved query",