Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-suggestions): stop refreshes when accepting a deprecate (#121459)

Thiago Salvatoreb93a8f0085da
diff --git a/frontend/src/scenes/data-warehouse/saved_queries/activityDescriptions.test.tsx b/frontend/src/scenes/data-warehouse/saved_queries/activityDescriptions.test.tsx
index 3b33147283cb..c666b40fe711 100644
--- a/frontend/src/scenes/data-warehouse/saved_queries/activityDescriptions.test.tsx
+++ b/frontend/src/scenes/data-warehouse/saved_queries/activityDescriptions.test.tsx
@@ -7,14 +7,18 @@ import { ActivityScope } from '~/types'
import { dataWarehouseSavedQueryActivityDescriber } from './activityDescriptions'
-function describeText(changes: ActivityChange[]): string {
+function describeText(
+ changes: ActivityChange[],
+ activity: string = 'updated',
+ trigger: ActivityLogItem['detail']['trigger'] = null
+): string {
const logItem: ActivityLogItem = {
user: { first_name: 'Max', last_name: 'Hog', email: '[email protected]' },
- activity: 'updated',
+ activity,
created_at: '2026-03-01T00:00:00Z',
scope: ActivityScope.DATA_WAREHOUSE_SAVED_QUERY,
item_id: '7',
- detail: { name: 'daily view', changes, merge: null, trigger: null },
+ detail: { name: 'daily view', changes, merge: null, trigger },
}
const { description } = dataWarehouseSavedQueryActivityDescriber(logItem)
const { container } = render(<>{description}</>)
@@ -77,6 +81,15 @@ describe('dataWarehouseSavedQueryActivityDescriber', () => {
expect(describeText([change('incremental_config', before, after)])).toContain(expected)
})
+ it.each([
+ [{ job_type: 'warehouse_suggestion', job_id: 'suggestion-1', payload: { kind: 'deprecate' } }, true],
+ [null, false],
+ ])('says when a warehouse suggestion turned materialization off', (trigger, viaSuggestion) => {
+ expect(describeText([], 'materialization_disabled', trigger).includes('(via a warehouse suggestion)')).toBe(
+ viaSuggestion
+ )
+ })
+
it('keeps the existing activity branches', () => {
expect(describeText([change('sync_frequency_interval', '0:01:00', '1:00:00')])).toContain(
'changed sync frequency from 1 minute to 1 hour'
diff --git a/frontend/src/scenes/data-warehouse/saved_queries/activityDescriptions.tsx b/frontend/src/scenes/data-warehouse/saved_queries/activityDescriptions.tsx
index 1c5b59c63c65..7fd8756105bd 100644
--- a/frontend/src/scenes/data-warehouse/saved_queries/activityDescriptions.tsx
+++ b/frontend/src/scenes/data-warehouse/saved_queries/activityDescriptions.tsx
@@ -229,6 +229,8 @@ function describeSyncFrequencyReset(
}
}
+const WAREHOUSE_SUGGESTION_TRIGGER_JOB_TYPE = 'warehouse_suggestion'
+
const SAVED_QUERY_EVENTS = new Map([
['created', { action: 'Created the view', description: 'created' }],
['deleted', { action: 'Deleted the view', description: 'deleted' }],
@@ -246,7 +248,14 @@ export function dataWarehouseSavedQueryActivityDescriber(
return { description: null }
}
- const user = <ActivityLogUserName logItem={logItem} />
+ const user = (
+ <>
+ <ActivityLogUserName logItem={logItem} />
+ {logItem.detail?.trigger?.job_type === WAREHOUSE_SUGGESTION_TRIGGER_JOB_TYPE ? (
+ <span className="text-muted"> (via a warehouse suggestion)</span>
+ ) : null}
+ </>
+ )
const viewName = logItem.detail?.name ? <strong>{logItem.detail.name}</strong> : <i>a view</i>
const event = SAVED_QUERY_EVENTS.get(logItem.activity)
if (event) {
diff --git a/products/cdp/backend/facade/api.py b/products/cdp/backend/facade/api.py
index 2b2de23d601c..c8f0e1f1971e 100644
--- a/products/cdp/backend/facade/api.py
+++ b/products/cdp/backend/facade/api.py
@@ -7,21 +7,32 @@ keep the DRF/serializer import chain off config-only import paths.
from __future__ import annotations
-from collections.abc import Sequence
+import operator
+from collections.abc import Collection, Sequence
+from functools import reduce
from typing import Any
from uuid import UUID
+from django.db.models import Q, QuerySet
+
from posthog.models.team import Team
from posthog.models.user import User
from products.cdp.backend.flag_gated_templates import FLAG_GATED_TEMPLATE_IDS, gated_template_enabled
from products.cdp.backend.models.hog_function_template import HogFunctionTemplate
+from products.cdp.backend.models.hog_functions import HogFunction
_B = "products.cdp.backend."
_LAZY = {"HogFunctionSerializer": "api.hog_function"}
-__all__ = [*sorted(_LAZY), "create_hog_functions", "is_hog_function_template_available"]
+__all__ = [
+ *sorted(_LAZY),
+ "create_hog_functions",
+ "has_enabled_hog_function_for_warehouse_table",
+ "is_hog_function_template_available",
+ "warehouse_tables_with_enabled_hog_functions",
+]
def is_hog_function_template_available(template_id: str, team: Team) -> bool:
@@ -65,6 +76,42 @@ def create_hog_functions(
return created_ids
+def _enabled_hog_functions_triggered_by(
+ team_id: int, trigger_source: str, table_names: Collection[str]
+) -> QuerySet[HogFunction]:
+ names_any_table = reduce(
+ operator.or_, (Q(filters__data_warehouse__contains=[{"table_name": name}]) for name in table_names)
+ )
+ return (
+ HogFunction.objects.filter(team_id=team_id, enabled=True, filters__source=trigger_source)
+ .filter(names_any_table)
+ .exclude(deleted=True)
+ )
+
+
+def has_enabled_hog_function_for_warehouse_table(*, team_id: int, trigger_source: str, table_name: str) -> bool:
+ """Whether an enabled hog function triggers on this warehouse table."""
+ return _enabled_hog_functions_triggered_by(team_id, trigger_source, [table_name]).exists()
+
+
+def warehouse_tables_with_enabled_hog_functions(
+ *, team_id: int, trigger_source: str, table_names: Collection[str]
+) -> set[str]:
+ """The given warehouse table names that an enabled hog function triggers on."""
+ names = set(table_names)
+ if not names:
+ return set()
+ triggering_filters = _enabled_hog_functions_triggered_by(team_id, trigger_source, names).values_list(
+ "filters", flat=True
+ )
+ return {
+ table["table_name"]
+ for filters in triggering_filters
+ for table in (filters or {}).get("data_warehouse") or []
+ if isinstance(table, dict) and isinstance(table.get("table_name"), str) and table["table_name"] in names
+ }
+
+
def __getattr__(name: str):
module = _LAZY.get(name)
if module is None:
diff --git a/products/cdp/backend/test/test_facade.py b/products/cdp/backend/test/test_facade.py
new file mode 100644
index 000000000000..89efda0c27a9
--- /dev/null
+++ b/products/cdp/backend/test/test_facade.py
@@ -0,0 +1,31 @@
+from posthog.test.base import BaseTest
+
+from products.cdp.backend.facade.api import warehouse_tables_with_enabled_hog_functions
+from products.cdp.backend.models.hog_functions import HogFunction
+
+VIEW_SOURCE = "data-warehouse-view"
+
+
+class TestWarehouseTablesWithEnabledHogFunctions(BaseTest):
+ def _function(self, source: str, table_names: list[str], *, enabled: bool = True, deleted: bool = False) -> None:
+ HogFunction.objects.create(
+ team=self.team,
+ enabled=enabled,
+ deleted=deleted,
+ filters={"source": source, "data_warehouse": [{"table_name": name} for name in table_names]},
+ )
+
+ def test_returns_only_the_asked_names_an_enabled_function_of_that_source_triggers_on(self) -> None:
+ self._function(VIEW_SOURCE, ["orders", "refunds", "not_asked"])
+ self._function(VIEW_SOURCE, ["invoices"])
+ self._function(VIEW_SOURCE, ["disabled_view"], enabled=False)
+ self._function(VIEW_SOURCE, ["deleted_view"], deleted=True)
+ self._function("data-warehouse-table", ["table_source"])
+
+ triggered = warehouse_tables_with_enabled_hog_functions(
+ team_id=self.team.pk,
+ trigger_source=VIEW_SOURCE,
+ table_names=["orders", "refunds", "invoices", "disabled_view", "deleted_view", "table_source", "unused"],
+ )
+
+ assert triggered == {"orders", "refunds", "invoices"}
diff --git a/products/customer_analytics/backend/apps.py b/products/customer_analytics/backend/apps.py
index 081da8eebc6c..573d00af97b7 100644
--- a/products/customer_analytics/backend/apps.py
+++ b/products/customer_analytics/backend/apps.py
@@ -73,6 +73,7 @@ class CustomerAnalyticsConfig(AppConfig):
register_person_property_projection,
register_person_property_sync_recorder,
register_person_property_sync_sources,
+ register_saved_query_property_sources,
)
def _projection_resolver(team_id: int, schema_id) -> list[PersonPropertySourceProjection] | None:
@@ -94,6 +95,14 @@ class CustomerAnalyticsConfig(AppConfig):
record_sync_run(record)
+ def _saved_query_sources_resolver(team_id: int, saved_query_ids) -> set[str]:
+ from products.customer_analytics.backend.logic.person_property_projection import ( # noqa: PLC0415
+ saved_queries_with_property_sources,
+ )
+
+ return saved_queries_with_property_sources(team_id, saved_query_ids)
+
register_person_property_projection(_projection_resolver)
register_person_property_sync_sources(_sync_sources_resolver)
register_person_property_sync_recorder(_run_recorder)
+ register_saved_query_property_sources(_saved_query_sources_resolver)
diff --git a/products/customer_analytics/backend/logic/person_property_projection.py b/products/customer_analytics/backend/logic/person_property_projection.py
index 200ff2a81d79..d41c55fd7e20 100644
--- a/products/customer_analytics/backend/logic/person_property_projection.py
+++ b/products/customer_analytics/backend/logic/person_property_projection.py
@@ -11,6 +11,8 @@ once a binding is confirmed to have enabled person sources, so unconfigured tabl
vast majority of runs) never pay a flag-service call.
"""
+from collections.abc import Collection
+
from django.conf import settings
import posthoganalytics
@@ -127,3 +129,13 @@ def _property_descriptions(source: CustomPropertySource) -> dict[str, str]:
for column, description in column_descriptions.items()
if column in column_property_map and description
}
+
+
+def saved_queries_with_property_sources(team_id: int, saved_query_ids: Collection[str]) -> set[str]:
+ """The given saved queries that an enabled person, group or account property source reads, whatever the rollout flags say."""
+ return {
+ str(saved_query_id)
+ for saved_query_id in CustomPropertySource.objects.for_team(team_id)
+ .filter(is_enabled=True, saved_query_id__in=saved_query_ids)
+ .values_list("saved_query_id", flat=True)
+ }
diff --git a/products/customer_analytics/backend/logic/person_property_projection_test.py b/products/customer_analytics/backend/logic/person_property_projection_test.py
index 8d5dcf733a9f..ef77b38a531a 100644
--- a/products/customer_analytics/backend/logic/person_property_projection_test.py
+++ b/products/customer_analytics/backend/logic/person_property_projection_test.py
@@ -9,6 +9,7 @@ from products.customer_analytics.backend.logic.person_property_projection import
person_properties_flag_enabled,
person_property_projection,
person_property_sync_sources,
+ saved_queries_with_property_sources,
)
from products.customer_analytics.backend.models import CustomPropertySource, TargetType
from products.customer_analytics.backend.models.team_scoped_test_base import TeamScopedTestMixin
@@ -219,3 +220,37 @@ class PersonPropertyProjectionTest(TeamScopedTestMixin, BaseTest):
return_value=False,
):
assert resolver(self.team.id, self.binding) is None
+
+
+class SavedQueriesWithPropertySourcesTest(TeamScopedTestMixin, BaseTest):
+ def _source(self, saved_query, target_type: str, *, is_enabled: bool = True) -> None:
+ definition = create_custom_property_definition(
+ team_id=self.team.id, name=f"{saved_query.name} {target_type}", target_type=target_type
+ )
+ CustomPropertySource.objects.for_team(self.team.id).create(
+ team_id=self.team.id,
+ definition=definition,
+ saved_query=saved_query,
+ key_column="distinct_id",
+ column_property_map={"plan": "plan_tier"},
+ is_enabled=is_enabled,
+ )
+
+ @patch(
+ "products.customer_analytics.backend.logic.person_property_projection.person_properties_flag_enabled",
+ return_value=False,
+ )
+ def test_finds_views_with_an_enabled_source_whatever_the_rollout_flag_says(self, _flag) -> None:
+ person = create_saved_query(team_id=self.team.id, name="person_view")
+ account = create_saved_query(team_id=self.team.id, name="account_view")
+ disabled = create_saved_query(team_id=self.team.id, name="disabled_view")
+ unused = create_saved_query(team_id=self.team.id, name="unused_view")
+ self._source(person, TargetType.PERSON.value)
+ self._source(account, TargetType.ACCOUNT.value)
+ self._source(disabled, TargetType.PERSON.value, is_enabled=False)
+
+ found = saved_queries_with_property_sources(
+ self.team.id, [str(view.id) for view in (person, account, disabled, unused)]
+ )
+
+ assert found == {str(person.id), str(account.id)}
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index 890c60215bc1..69fb2ef3da0b 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -57,8 +57,10 @@ _LAZY = {
"MaterializationRefusedError": "facade.contracts",
"IncrementalConfigRefusedError": "facade.contracts",
"enable_saved_query_materialization": "logic.saved_query_materialization",
+ "disable_saved_query_materialization": "logic.saved_query_materialization",
"link_table_if_materializing": "logic.saved_query_dag_sync",
"unlink_table": "logic.saved_query_dag_sync",
+ "saved_query_still_materializes": "logic.saved_query_materialization",
"saved_query_materialized_at": "logic.saved_query_freshness",
"latest_saved_query_materialization_job": "logic.saved_query_freshness",
"is_materialization_fresh": "logic.saved_query_freshness",
diff --git a/products/data_modeling/backend/logic/saved_query_materialization.py b/products/data_modeling/backend/logic/saved_query_materialization.py
index ffa60d8c0bbd..d4bf4cae44dc 100644
--- a/products/data_modeling/backend/logic/saved_query_materialization.py
+++ b/products/data_modeling/backend/logic/saved_query_materialization.py
@@ -4,13 +4,14 @@ from enum import Enum
from typing import Any, Literal
from uuid import UUID
+from django.db import transaction
from django.db.models import Q, QuerySet
import structlog
from posthog.exceptions_capture import capture_exception
from posthog.models import User
-from posthog.models.activity_logging.activity_log import Change, Detail, log_activity
+from posthog.models.activity_logging.activity_log import Change, Detail, Trigger, log_activity
from posthog.rbac.query_access import assert_user_can_read_query
from products.access_control.backend.facade.user_access_control import UserAccessControl
@@ -26,19 +27,28 @@ from products.data_modeling.backend.logic.incremental_eligibility import (
check_incremental_eligibility,
definition_database,
)
-from products.data_modeling.backend.logic.node_frequency import SavedQueryFrequencyBounds, saved_query_target_bounds
+from products.data_modeling.backend.logic.node_frequency import (
+ SavedQueryFrequencyBounds,
+ declared_targets_from_nodes,
+ saved_query_target_bounds,
+)
from products.data_modeling.backend.logic.node_materialization import SavedQueryNotFoundError
-from products.data_modeling.backend.logic.saved_query_dag_sync import update_node_type
+from products.data_modeling.backend.logic.saved_query_dag_sync import materializes, update_node_type
from products.data_modeling.backend.logic.saved_query_reads import get_saved_query_columns
from products.data_modeling.backend.logic.schedule_reconcile import check_saved_query_frequency_target
from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
-from products.data_modeling.backend.models.node import NodeType
+from products.data_modeling.backend.models.node import Node, NodeType
logger = structlog.get_logger(__name__)
VisibleBlockerNames = Callable[[SavedQueryFrequencyBounds], Mapping[str, str]]
MANAGED_VIEWSET_REFUSAL = "Cannot materialize a query from a managed viewset."
+MANAGED_VIEWSET_REVERT_REFUSAL = "Cannot revert materialization of a query from a managed viewset."
+STILL_MATERIALIZING_MESSAGE = (
+ "Materialization could not be fully turned off: the view is still scheduled to refresh. Try again."
+)
+MATERIALIZATION_DISABLED_ACTIVITY = "materialization_disabled"
EDIT_ACCESS_REFUSAL = "You need edit access to this view to materialize it."
LINEAGE_CHANGED_REFUSAL = "This view's lineage changed while we were setting it up. Reopen it and pick a cadence again."
MATERIALIZATION_FAILED_MESSAGE = "Materialization failed. Please try again or contact support."
@@ -64,13 +74,10 @@ def enable_saved_query_materialization(
visible_blocker_names: VisibleBlockerNames,
was_impersonated: bool,
incremental_config: IncrementalConfigUpdate = KEEP_INCREMENTAL_CONFIG,
+ trigger: Trigger | None = None,
) -> None:
"""Materialize a saved query at the given cadence and start its first run."""
- saved_query = (
- DataWarehouseSavedQuery.objects.filter(team_id=team_id, id=saved_query_id).exclude(deleted=True).first()
- )
- if saved_query is None:
- raise SavedQueryNotFoundError(f"Saved query {saved_query_id} not found for team {team_id}")
+ saved_query = _live_saved_query(team_id, saved_query_id)
_enable_materialization(
saved_query,
user=user,
@@ -78,6 +85,71 @@ def enable_saved_query_materialization(
visible_blocker_names=visible_blocker_names,
was_impersonated=was_impersonated,
incremental_config=incremental_config,
+ trigger=trigger,
+ )
+
+
+def disable_saved_query_materialization(
+ team_id: int, saved_query_id: UUID | str, *, user: User, was_impersonated: bool, trigger: Trigger | None = None
+) -> None:
+ """Turn a materialized saved query back into a plain view: drop its table and its refresh schedule."""
+ saved_query = _live_saved_query(team_id, saved_query_id)
+ if saved_query.managed_viewset_id is not None:
+ raise MaterializationRefusedError(MANAGED_VIEWSET_REVERT_REFUSAL)
+ _require_edit_access(saved_query, user)
+
+ with transaction.atomic():
+ saved_query.revert_materialization(delete_joins=False)
+ _mark_nodes_ephemeral(saved_query)
+ saved_query.refresh_from_db()
+ if _still_materializing(saved_query):
+ raise MaterializationFailedError(STILL_MATERIALIZING_MESSAGE)
+
+ log_activity(
+ organization_id=saved_query.team.organization_id,
+ team_id=saved_query.team_id,
+ user=user,
+ was_impersonated=was_impersonated,
+ item_id=saved_query.id,
+ scope=SAVED_QUERY_ACTIVITY_SCOPE,
+ activity=MATERIALIZATION_DISABLED_ACTIVITY,
+ detail=Detail(name=saved_query.name, trigger=trigger),
+ )
+
+
+def _live_saved_query(team_id: int, saved_query_id: UUID | str) -> DataWarehouseSavedQuery:
+ saved_query = (
+ DataWarehouseSavedQuery.objects.filter(team_id=team_id, id=saved_query_id).exclude(deleted=True).first()
+ )
+ if saved_query is None:
+ raise SavedQueryNotFoundError(f"Saved query {saved_query_id} not found for team {team_id}")
+ return saved_query
+
+
+def _mark_nodes_ephemeral(saved_query: DataWarehouseSavedQuery) -> None:
+ # set data modeling node type to view
+ try:
+ update_node_type(saved_query, NodeType.VIEW)
+ except Exception as e:
+ capture_exception(e)
+ logger.exception("Failed to update node type to view", saved_query_name=saved_query.name)
+ raise MaterializationFailedError(STILL_MATERIALIZING_MESSAGE) from e
+
+
+def saved_query_still_materializes(team_id: int, saved_query_id: UUID | str) -> bool:
+ """Whether the saved query asks to materialize, has a matview node, or keeps a frequency target."""
+ try:
+ return _still_materializing(_live_saved_query(team_id, saved_query_id))
+ except SavedQueryNotFoundError:
+ return False
+
+
+def _still_materializing(saved_query: DataWarehouseSavedQuery) -> bool:
+ nodes = list(Node.objects.filter(team_id=saved_query.team_id, saved_query=saved_query))
+ return (
+ materializes(saved_query)
+ or any(node.type == NodeType.MAT_VIEW for node in nodes)
+ or bool(declared_targets_from_nodes(nodes))
)
@@ -89,6 +161,7 @@ def _enable_materialization(
visible_blocker_names: VisibleBlockerNames,
was_impersonated: bool,
incremental_config: IncrementalConfigUpdate,
+ trigger: Trigger | None,
) -> None:
if saved_query.managed_viewset_id is not None:
raise MaterializationRefusedError(MANAGED_VIEWSET_REFUSAL)
@@ -138,6 +211,7 @@ def _enable_materialization(
was_impersonated=was_impersonated,
previous_interval=previous_interval,
sync_frequency_interval=sync_frequency_interval,
+ trigger=trigger,
)
@@ -231,6 +305,7 @@ def _log_materialization_enabled(
was_impersonated: bool,
previous_interval: timedelta | None,
sync_frequency_interval: timedelta | None,
+ trigger: Trigger | None,
) -> None:
log_activity(
organization_id=saved_query.team.organization_id,
@@ -251,5 +326,6 @@ def _log_materialization_enabled(
after=str(sync_frequency_interval),
),
],
+ trigger=trigger,
),
)
diff --git a/products/data_modeling/backend/logic/saved_query_reads.py b/products/data_modeling/backend/logic/saved_query_reads.py
index 88cdb63ddace..c224d874a609 100644
--- a/products/data_modeling/backend/logic/saved_query_reads.py
+++ b/products/data_modeling/backend/logic/saved_query_reads.py
@@ -95,11 +95,16 @@ def all_saved_query_names(team_id: int) -> dict[str, str]:
return {str(saved_query_id): name for saved_query_id, name in rows}
-def saved_query_definitions(team_id: int) -> list[SavedQueryDefinition]:
- """Every saved query in this team that still resolves, with its HogQL and whether its DAG node materializes."""
+def saved_query_definitions(team_id: int, *, ids: Collection[UUID] | None = None) -> list[SavedQueryDefinition]:
+ """Every saved query in this team that still resolves, with its HogQL and whether its DAG node materializes.
+
+ ``ids`` narrows the result to those saved queries.
+ """
+ saved_queries_in_team = DataWarehouseSavedQuery.objects.filter(team_id=team_id)
+ if ids is not None:
+ saved_queries_in_team = saved_queries_in_team.filter(id__in=ids)
saved_queries = list(
- DataWarehouseSavedQuery.objects.filter(team_id=team_id)
- .exclude(deleted=True)
+ saved_queries_in_team.exclude(deleted=True)
.only("id", "name", "query", "is_test", "managed_viewset_id", "origin", "created_at")
.order_by("created_at", "id")
)
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py b/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
index 61bf9eb5a7ab..572cd48ad644 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
@@ -17,7 +17,6 @@ from posthog.hogql.database.database import Database
from posthog.api.mixins import ValidatedRequest, validated_request
from posthog.api.routing import TeamAndOrgViewSetMixin
-from posthog.exceptions_capture import capture_exception
from posthog.helpers.impersonation import is_impersonated
from posthog.models import User
from posthog.models.activity_logging.activity_log import Detail, load_activity, log_activity
@@ -31,6 +30,7 @@ from products.data_modeling.backend.facade.api import (
MaterializationForbiddenError,
MaterializationRefusedError,
SavedQueryNotFoundError,
+ disable_saved_query_materialization,
enable_saved_query_materialization,
)
from products.data_modeling.backend.facade.flags import is_managed_viewset_enabled
@@ -379,31 +379,19 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
"""
saved_query: DataWarehouseSavedQuery = self.get_object()
- if saved_query.managed_viewset is not None:
- raise serializers.ValidationError("Cannot revert materialization of a query from a managed viewset.")
-
- saved_query.revert_materialization()
-
- # set data modeling node type to view
try:
- from products.data_modeling.backend.facade.api import update_node_type
- from products.data_modeling.backend.facade.models import NodeType
-
- update_node_type(saved_query, NodeType.VIEW)
- except Exception as e:
- capture_exception(e)
- logger.exception("Failed to update node type to view", saved_query_name=saved_query.name)
-
- log_activity(
- organization_id=self.team.organization_id,
- team_id=self.team_id,
- user=cast(User, request.user),
- was_impersonated=is_impersonated(request),
- item_id=saved_query.id,
- scope="DataWarehouseSavedQuery",
- activity="materialization_disabled",
- detail=Detail(name=saved_query.name),
- )
+ disable_saved_query_materialization(
+ self.team_id,
+ saved_query.id,
+ user=cast(User, request.user),
+ was_impersonated=is_impersonated(request),
+ )
+ except SavedQueryNotFoundError:
+ raise exceptions.NotFound()
+ except MaterializationForbiddenError as e:
+ raise exceptions.PermissionDenied(str(e))
+ except (MaterializationRefusedError, MaterializationFailedError) as e:
+ raise serializers.ValidationError(str(e))
return response.Response(status=status.HTTP_200_OK)
diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.py
index 53957a2373c6..576a2757b5bc 100644
--- a/products/data_warehouse/backend/tests/api/test_saved_query.py
+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py
@@ -33,6 +33,7 @@ from products.data_modeling.backend.facade.models import (
Node,
NodeType,
)
+from products.data_tools.backend.facade.models import DataWarehouseJoin
from products.data_tools.backend.models.datawarehouse_saved_query_folder import DataWarehouseSavedQueryFolder
from products.data_warehouse.backend.presentation.views.saved_query.editing import (
INCREMENTAL_CONFIG_CHANGED_REFUSAL,
@@ -2056,16 +2057,33 @@ class TestSavedQuery(APIBaseTest):
db_saved_query.is_materialized = True
mock_table = DataWarehouseTable.objects.create(
- team=self.team, name="materialized_event_view", format="Parquet", url_pattern="s3://bucket/path"
+ team=self.team, name="event_view", format="Parquet", url_pattern="s3://bucket/path"
)
db_saved_query.table = mock_table
db_saved_query.save()
+ Node.objects.filter(saved_query=db_saved_query).update(type=NodeType.MAT_VIEW)
+ view_join = DataWarehouseJoin.objects.create(
+ team=self.team,
+ source_table_name="event_view",
+ source_table_key="event",
+ joining_table_name="events",
+ joining_table_key="event",
+ field_name="events",
+ )
response = self.client.post(
f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query_id}/revert_materialization",
)
self.assertEqual(response.status_code, 200, response.content)
+ self.assertEqual(
+ list(Node.objects.filter(saved_query=db_saved_query).values_list("type", flat=True)), [NodeType.VIEW]
+ )
+ self.assertTrue(
+ ActivityLog.objects.filter(
+ team_id=self.team.id, item_id=saved_query_id, activity="materialization_disabled"
+ ).exists()
+ )
db_saved_query.refresh_from_db()
self.assertIsNone(db_saved_query.sync_frequency_interval)
@@ -2078,6 +2096,8 @@ class TestSavedQuery(APIBaseTest):
# Check the table has been deleted
mock_table.refresh_from_db()
self.assertTrue(mock_table.deleted)
+ view_join.refresh_from_db()
+ self.assertFalse(view_join.deleted)
def test_create_with_existing_name(self):
DataWarehouseTable.objects.create(
diff --git a/products/warehouse_sources/backend/facade/api.py b/products/warehouse_sources/backend/facade/api.py
index e965ae0a9aec..85f968c850bd 100644
--- a/products/warehouse_sources/backend/facade/api.py
+++ b/products/warehouse_sources/backend/facade/api.py
@@ -68,6 +68,7 @@ from products.warehouse_sources.backend.models.util import (
)
from products.warehouse_sources.backend.source_status import effective_source_status
from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.backend.view_consumers import saved_queries_feeding_consumers
from . import contracts
@@ -96,6 +97,7 @@ __all__ = [
"list_tables_for_source",
"list_jobs_for_source",
"list_column_statistics",
+ "saved_queries_feeding_consumers",
# framework-free helper transforms
"effective_source_status",
"clickhouse_columns_to_dwh_columns",
diff --git a/products/warehouse_sources/backend/facade/hooks.py b/products/warehouse_sources/backend/facade/hooks.py
index 67f03dec03f3..500a1c184f41 100644
--- a/products/warehouse_sources/backend/facade/hooks.py
+++ b/products/warehouse_sources/backend/facade/hooks.py
@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.external_product_h
register_person_property_sync_recorder,
register_person_property_sync_sources,
register_revenue_view_sync,
+ register_saved_query_property_sources,
run_revenue_view_sync,
saved_query_binding,
schema_binding,
@@ -58,6 +59,7 @@ __all__ = [
"register_person_property_sync_recorder",
"register_person_property_sync_sources",
"register_revenue_view_sync",
+ "register_saved_query_property_sources",
"run_revenue_view_sync",
"saved_query_binding",
"schema_binding",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py b/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py
index ecbb5d36c5d6..d68b2fdba590 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_product_hooks.py
@@ -11,7 +11,7 @@ pipeline degrades to a no-op, which keeps warehouse_sources importable on its ow
import uuid
import dataclasses
-from collections.abc import Callable
+from collections.abc import Callable, Collection
from typing import TYPE_CHECKING, Any, Optional
from products.warehouse_sources.backend.facade.contracts import RevenueViewSyncInput
@@ -187,6 +187,22 @@ def person_property_sync_enabled_for(team_id: int, binding: WarehouseBinding) ->
return person_property_projection_for(team_id, binding) is not None
+SavedQueryPropertySourcesResolver = Callable[[int, Collection[str]], set[str]]
+_saved_query_property_sources_resolver: Optional[SavedQueryPropertySourcesResolver] = None
+
+
+def register_saved_query_property_sources(fn: SavedQueryPropertySourcesResolver) -> None:
+ global _saved_query_property_sources_resolver
+ _saved_query_property_sources_resolver = fn
+
+
+def saved_queries_with_property_sources(team_id: int, saved_query_ids: Collection[str]) -> set[str]:
+ """The given saved queries that a configured property source reads, whether or not its sync runs today."""
+ if _saved_query_property_sources_resolver is None:
+ return set()
+ return _saved_query_property_sources_resolver(team_id, saved_query_ids)
+
+
@dataclasses.dataclass(frozen=True)
class AccountPropertySourceProjection:
"""One account-target source's materialization projection."""
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py
index 68cab77d39c2..9f909f7d4db1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py
@@ -23,7 +23,7 @@ from posthog.kafka_client.routing import KafkaClusterProfile, async_producer_sco
from posthog.kafka_client.topics import KAFKA_DWH_CDP_RAW_TABLE
from posthog.sync import database_sync_to_async_pool
-from products.cdp.backend.models.hog_functions import HogFunction
+from products.cdp.backend.facade.api import has_enabled_hog_function_for_warehouse_table
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.data_warehouse.backend.facade.api import aget_s3_client, ensure_bucket_exists
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
@@ -39,6 +39,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.sta
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name
from products.warehouse_sources.backend.temporal.data_imports.util import PostHogInternalDatabaseError
+from products.warehouse_sources.backend.view_consumers import VIEW_TRIGGER_SOURCE
from products.workflows.backend.facade.api import has_active_workflow_for_warehouse_table
# Per-file exceptions are swallowed (the file is deleted and the run continues), so a failed file
@@ -61,7 +62,7 @@ TableKind = typing.Literal["source", "view"]
# keyed by the kind of warehouse table the rows came from.
TRIGGER_SOURCE_BY_KIND: dict[TableKind, str] = {
"source": "data-warehouse-table",
- "view": "data-warehouse-view",
+ "view": VIEW_TRIGGER_SOURCE,
}
@@ -242,18 +243,9 @@ class CDPProducer:
if self.table.kind == "view" and not self._view_can_trigger():
return False
- has_matching_hog_function = (
- HogFunction.objects.filter(
- team_id=self.team_id,
- enabled=True,
- filters__source=trigger_source,
- filters__data_warehouse__contains=[{"table_name": dot_notated_table_name}],
- )
- .exclude(deleted=True)
- .exists()
- )
-
- if has_matching_hog_function:
+ if has_enabled_hog_function_for_warehouse_table(
+ team_id=self.team_id, trigger_source=trigger_source, table_name=dot_notated_table_name
+ ):
return True
# Also gate on active workflows (HogFlows) triggered by this table - without this the
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py
index 56b8f4b6c2c4..971636ac9f93 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py
@@ -349,9 +349,9 @@ async def test_should_run_posthog_database_connection_failure_stays_retryable(
producer = CDPProducer.for_source(team_id=team.id, schema_id=str(schema.id), job_id="", logger=mock.AsyncMock())
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.HogFunction.objects"
- ) as mock_hog_function_objects:
- mock_hog_function_objects.filter.side_effect = error_cls(message)
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.has_enabled_hog_function_for_warehouse_table",
+ side_effect=error_cls(message),
+ ):
with pytest.raises(PostHogInternalDatabaseError) as exc_info:
await producer.should_run()
diff --git a/products/warehouse_sources/backend/tests/test_view_consumers.py b/products/warehouse_sources/backend/tests/test_view_consumers.py
new file mode 100644
index 000000000000..d57a8d09a4d3
--- /dev/null
+++ b/products/warehouse_sources/backend/tests/test_view_consumers.py
@@ -0,0 +1,59 @@
+from collections.abc import Callable
+
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from parameterized import parameterized
+
+from products.cdp.backend.models.hog_functions import HogFunction
+from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
+from products.warehouse_sources.backend.facade.api import saved_queries_feeding_consumers
+from products.workflows.backend.facade.testing import create_workflow_for_test
+
+VIEW_NAME = "daily_revenue"
+PROPERTY_SOURCES = "products.warehouse_sources.backend.view_consumers.saved_queries_with_property_sources"
+
+
+def _hog_function(team_id: int) -> None:
+ HogFunction.objects.create(
+ team_id=team_id,
+ enabled=True,
+ filters={"source": "data-warehouse-view", "data_warehouse": [{"table_name": VIEW_NAME}]},
+ )
+
+
+def _workflow(team_id: int) -> None:
+ create_workflow_for_test(
+ team_id=team_id, status="active", trigger={"type": "data-warehouse-view", "table_name": VIEW_NAME}
+ )
+
+
+def _nothing(team_id: int) -> None:
+ pass
+
+
+class TestSavedQueriesFeedingConsumers(BaseTest):
+ @parameterized.expand(
+ [
+ ("hog_function", _hog_function, False, True),
+ ("workflow", _workflow, False, True),
+ ("property_source", _nothing, True, True),
+ ("no_consumer", _nothing, False, False),
+ ]
+ )
+ def test_finds_views_whose_refreshes_feed_a_consumer(
+ self,
+ _name: str,
+ add_consumer: Callable[[int], None],
+ has_property_source: bool,
+ feeds: bool,
+ ) -> None:
+ view = DataWarehouseSavedQuery.objects.create(
+ team=self.team, name=VIEW_NAME, query={"kind": "HogQLQuery", "query": "SELECT 1"}
+ )
+ add_consumer(self.team.pk)
+
+ with patch(PROPERTY_SOURCES, return_value={str(view.id)} if has_property_source else set()):
+ feeding = saved_queries_feeding_consumers(self.team.pk, [view.id])
+
+ assert (view.id in feeding) is feeds
diff --git a/products/warehouse_sources/backend/view_consumers.py b/products/warehouse_sources/backend/view_consumers.py
new file mode 100644
index 000000000000..360eb5e65802
--- /dev/null
+++ b/products/warehouse_sources/backend/view_consumers.py
@@ -0,0 +1,36 @@
+from collections.abc import Collection
+from uuid import UUID
+
+from products.cdp.backend.facade.api import warehouse_tables_with_enabled_hog_functions
+from products.data_modeling.backend.facade.api import all_saved_query_names
+from products.warehouse_sources.backend.temporal.data_imports.external_product_hooks import (
+ saved_queries_with_property_sources,
+)
+from products.workflows.backend.facade.api import active_workflow_warehouse_table_names
+
+VIEW_TRIGGER_SOURCE = "data-warehouse-view"
+
+
+def saved_queries_feeding_consumers(team_id: int, saved_query_ids: Collection[UUID]) -> frozenset[UUID]:
+ """The given saved queries whose refreshes feed person, group or account properties, a hog function or a workflow."""
+ team_names = all_saved_query_names(team_id)
+ names = {
+ saved_query_id: team_names[str(saved_query_id)]
+ for saved_query_id in saved_query_ids
+ if str(saved_query_id) in team_names
+ }
+ if not names:
+ return frozenset()
+ triggered_names = warehouse_tables_with_enabled_hog_functions(
+ team_id=team_id, trigger_source=VIEW_TRIGGER_SOURCE, table_names=names.values()
+ ) | active_workflow_warehouse_table_names(
+ team_id=team_id, trigger_source=VIEW_TRIGGER_SOURCE, table_names=names.values()
+ )
+ with_property_sources = saved_queries_with_property_sources(
+ team_id, [str(saved_query_id) for saved_query_id in names]
+ )
+ return frozenset(
+ saved_query_id
+ for saved_query_id, name in names.items()
+ if name in triggered_names or str(saved_query_id) in with_property_sources
+ )
diff --git a/products/warehouse_suggestions/backend/facade/api.py b/products/warehouse_suggestions/backend/facade/api.py
index 45e94a1af9fc..071db93be185 100644
--- a/products/warehouse_suggestions/backend/facade/api.py
+++ b/products/warehouse_suggestions/backend/facade/api.py
@@ -1,6 +1,7 @@
"""Facade for warehouse_suggestions."""
from collections.abc import Sequence
+from dataclasses import asdict
from typing import TYPE_CHECKING
from uuid import UUID
@@ -159,7 +160,7 @@ def accept_suggestion(
[outcome.suggestion],
team=team,
user=user,
- properties={"changed_fields": list(outcome.changed_fields)},
+ properties={"changed_fields": list(outcome.changed_fields), **asdict(outcome.savings)},
)
return _single_contract(team.pk, user_access_control, outcome.suggestion, access)
diff --git a/products/warehouse_suggestions/backend/facade/contracts.py b/products/warehouse_suggestions/backend/facade/contracts.py
index 8c299491b6ff..49ad6909e37d 100644
--- a/products/warehouse_suggestions/backend/facade/contracts.py
+++ b/products/warehouse_suggestions/backend/facade/contracts.py
@@ -143,6 +143,7 @@ class Suggestion:
@frozen
class CertificationAsset:
certification_id: str
+ stopped_materialization: bool = False
@frozen
@@ -164,6 +165,7 @@ class AcceptOptions:
incremental_key: str | None = None
unique_key: tuple[str, ...] | None = None
lookback_seconds: int | None = None
+ keep_materialized: bool | None = None
CreatedAsset = CertificationAsset | MaterializationAsset
diff --git a/products/warehouse_suggestions/backend/logic/accept.py b/products/warehouse_suggestions/backend/logic/accept.py
index eaa850dd5abc..196a6a78b409 100644
--- a/products/warehouse_suggestions/backend/logic/accept.py
+++ b/products/warehouse_suggestions/backend/logic/accept.py
@@ -2,10 +2,11 @@ from abc import ABC, abstractmethod
from collections.abc import Callable, Mapping
from dataclasses import asdict, fields
from datetime import timedelta
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING
from uuid import UUID
from posthog.dataclasses import frozen
+from posthog.models.activity_logging.activity_log import Trigger
from products.data_catalog.backend.facade.api import certifications_for_team, certify, deprecate, propose_certification
from products.data_catalog.backend.facade.enums import CertificationStatus
@@ -17,13 +18,18 @@ from products.data_modeling.backend.facade.api import (
MaterializationRefusedError,
SavedQueryNotFoundError,
definition_database,
+ dependent_saved_query_ids,
+ disable_saved_query_materialization,
enable_saved_query_materialization,
get_saved_query_columns,
get_saved_query_sql,
get_saved_query_summary,
propose_incremental_config,
+ saved_query_definitions,
+ saved_query_still_materializes,
)
-from products.warehouse_sources.backend.facade.api import get_queryable_table
+from products.data_modeling.backend.facade.contracts import SavedQueryDefinition
+from products.warehouse_sources.backend.facade.api import get_queryable_table, saved_queries_feeding_consumers
from ..facade.contracts import (
AcceptFailedError,
@@ -31,6 +37,7 @@ from ..facade.contracts import (
AcceptOptions,
CertificationAsset,
CreatedAsset,
+ DeprecatePayload,
MaterializationAsset,
MaterializePayload,
SubjectAlreadyCertifiedError,
@@ -67,7 +74,9 @@ class AcceptRequest:
class Acceptor(ABC):
- option_fields: ClassVar[frozenset[str]] = frozenset()
+ @property
+ def option_fields(self) -> frozenset[str]:
+ return frozenset()
@abstractmethod
def accept(
@@ -75,18 +84,32 @@ class Acceptor(ABC):
) -> CreatedAsset: ...
+KEEP_MATERIALIZED_FIELD = "keep_materialized"
+WAREHOUSE_SUGGESTION_TRIGGER_JOB_TYPE = "warehouse_suggestion"
+OTHER_VIEWS_READ_IT = "Other views now read from this view."
+SYNCS_READ_IT = "This view now feeds person properties, account properties, a destination or a workflow."
+
+
@frozen
class CertificationAcceptor(Acceptor):
status: CertificationStatus
apply: "Callable[[TableCertification, User], TableCertification]"
reuses_existing: bool
+ stops_materialization: bool = False
+
+ @property
+ def option_fields(self) -> frozenset[str]:
+ return frozenset({KEEP_MATERIALIZED_FIELD}) if self.stops_materialization else frozenset()
def accept(
self, suggestion: WarehouseSuggestion, payload: SuggestionPayload, request: AcceptRequest
) -> CreatedAsset:
+ stops = self.stops_materialization and _may_stop_materializing(suggestion, request.options)
+ if stops:
+ stop_materializing(suggestion, request)
certification = self._certification(suggestion, request)
self.apply(certification, request.user)
- return CertificationAsset(certification_id=str(certification.id))
+ return CertificationAsset(certification_id=str(certification.id), stopped_materialization=stops)
def _certification(self, suggestion: WarehouseSuggestion, request: AcceptRequest) -> "TableCertification":
subject_kind = WarehouseSuggestionSubjectKind(suggestion.subject_kind)
@@ -109,6 +132,54 @@ class CertificationAcceptor(Acceptor):
return existing
+def _may_stop_materializing(suggestion: WarehouseSuggestion, options: AcceptOptions) -> bool:
+ if suggestion.subject_kind != WarehouseSuggestionSubjectKind.SAVED_QUERY or options.keep_materialized:
+ return False
+ definition = _definition(suggestion)
+ return (
+ definition is not None
+ and saved_query_still_materializes(suggestion.team_id, definition.id)
+ and stop_blocker(suggestion.team_id, definition) is None
+ )
+
+
+def _definition(suggestion: WarehouseSuggestion) -> SavedQueryDefinition | None:
+ return next(iter(saved_query_definitions(suggestion.team_id, ids=[suggestion.subject_id])), None)
+
+
+def stop_blocker(team_id: int, definition: SavedQueryDefinition) -> str | None:
+ if dependent_saved_query_ids(team_id, [definition.id]).get(definition.id):
+ return OTHER_VIEWS_READ_IT
+ if definition.id in saved_queries_feeding_consumers(team_id, [definition.id]):
+ return SYNCS_READ_IT
+ return None
+
+
+def suggestion_trigger(suggestion: WarehouseSuggestion) -> Trigger:
+ return Trigger(
+ job_type=WAREHOUSE_SUGGESTION_TRIGGER_JOB_TYPE, job_id=str(suggestion.id), payload={"kind": suggestion.kind}
+ )
+
+
+def stop_materializing(suggestion: WarehouseSuggestion, request: AcceptRequest) -> None:
+ try:
+ disable_saved_query_materialization(
+ suggestion.team_id,
+ suggestion.subject_id,
+ user=request.user,
+ was_impersonated=request.was_impersonated,
+ trigger=suggestion_trigger(suggestion),
+ )
+ except SavedQueryNotFoundError:
+ raise SuggestionSubjectGoneError(WarehouseSuggestionSubjectKind.SAVED_QUERY)
+ except MaterializationForbiddenError:
+ raise SubjectEditAccessRequiredError(WarehouseSuggestionSubjectKind.SAVED_QUERY)
+ except MaterializationRefusedError as error:
+ raise AcceptOptionRefusedError(KEEP_MATERIALIZED_FIELD, str(error))
+ except MaterializationFailedError as error:
+ raise AcceptFailedError(str(error))
+
+
VIEW_CHANGED_SINCE_SUGGESTED = (
"This view changed after the suggestion was made, so it can no longer refresh incrementally on the suggested keys."
)
@@ -127,7 +198,9 @@ UNIQUE_KEY_FIELD = "unique_key"
class MaterializeAcceptor(Acceptor):
- option_fields = frozenset(field.name for field in fields(AcceptOptions))
+ @property
+ def option_fields(self) -> frozenset[str]:
+ return frozenset(field.name for field in fields(AcceptOptions)) - {KEEP_MATERIALIZED_FIELD}
def accept(
self, suggestion: WarehouseSuggestion, payload: SuggestionPayload, request: AcceptRequest
@@ -152,6 +225,7 @@ class MaterializeAcceptor(Acceptor):
visible_blocker_names=_no_blocker_names,
was_impersonated=request.was_impersonated,
incremental_config=incremental_config,
+ trigger=suggestion_trigger(suggestion),
)
except SavedQueryNotFoundError:
raise SuggestionSubjectGoneError(WarehouseSuggestionSubjectKind.SAVED_QUERY)
@@ -236,7 +310,7 @@ ACCEPTORS: Mapping[WarehouseSuggestionKind, Acceptor] = {
status=CertificationStatus.CERTIFIED, apply=certify, reuses_existing=False
),
WarehouseSuggestionKind.DEPRECATE: CertificationAcceptor(
- status=CertificationStatus.DEPRECATED, apply=deprecate, reuses_existing=True
+ status=CertificationStatus.DEPRECATED, apply=deprecate, reuses_existing=True, stops_materialization=True
),
WarehouseSuggestionKind.MATERIALIZE: MaterializeAcceptor(),
}
@@ -245,11 +319,24 @@ ACCEPTORS: Mapping[WarehouseSuggestionKind, Acceptor] = {
RELEASE_TRANSITIONS: Transitions = {WarehouseSuggestionStatus.ACCEPTED: frozenset({WarehouseSuggestionStatus.PROPOSED})}
+@frozen
+class RefreshSavings:
+ stopped_materialization: bool
+ refresh_seconds_per_month: float
+ refresh_bytes_per_month: float
+
+
+NO_REFRESH_SAVINGS = RefreshSavings(
+ stopped_materialization=False, refresh_seconds_per_month=0, refresh_bytes_per_month=0
+)
+
+
@frozen
class AcceptOutcome:
suggestion: WarehouseSuggestion
newly_accepted: bool
changed_fields: tuple[str, ...] = ()
+ savings: RefreshSavings = NO_REFRESH_SAVINGS
def accept(team_id: int, suggestion_id: UUID, request: AcceptRequest) -> AcceptOutcome:
@@ -305,7 +392,22 @@ def _accept(team_id: int, suggestion_id: UUID, request: AcceptRequest) -> Accept
claimed.created_asset = asdict(created_asset)
claimed.save(update_fields=["created_asset"])
return AcceptOutcome(
- suggestion=claimed, newly_accepted=True, changed_fields=changed_fields(request.options, payload)
+ suggestion=claimed,
+ newly_accepted=True,
+ changed_fields=changed_fields(request.options, payload),
+ savings=refresh_savings(created_asset, payload),
+ )
+
+
+def refresh_savings(created_asset: CreatedAsset, payload: SuggestionPayload) -> RefreshSavings:
+ if not isinstance(created_asset, CertificationAsset) or not created_asset.stopped_materialization:
+ return NO_REFRESH_SAVINGS
+ if not isinstance(payload, DeprecatePayload):
+ return NO_REFRESH_SAVINGS
+ return RefreshSavings(
+ stopped_materialization=True,
+ refresh_seconds_per_month=payload.refresh_seconds_per_month,
+ refresh_bytes_per_month=payload.refresh_bytes_per_month,
)
diff --git a/products/warehouse_suggestions/backend/logic/candidates/deprecate.py b/products/warehouse_suggestions/backend/logic/candidates/deprecate.py
index d35285a7373a..3418c842d253 100644
--- a/products/warehouse_suggestions/backend/logic/candidates/deprecate.py
+++ b/products/warehouse_suggestions/backend/logic/candidates/deprecate.py
@@ -4,6 +4,7 @@ from uuid import UUID
from products.data_catalog.backend.facade.enums import CertificationStatus
from products.data_modeling.backend.facade.api import dependent_saved_query_ids, upstream_table_refs
from products.data_modeling.backend.facade.contracts import SavedQueryDefinition
+from products.warehouse_sources.backend.facade.api import saved_queries_feeding_consumers
from ...facade.contracts import DeprecatePayload
from ...facade.enums import WarehouseSuggestionAssetOutcome, WarehouseSuggestionKind, WarehouseSuggestionSubjectKind
@@ -38,12 +39,19 @@ class DeprecateCandidate(Candidate):
and context.is_suggestible_view(saved_query)
and not _is_read(context, _subject(saved_query))
]
- dependents = dependent_saved_query_ids(context.team_id, [saved_query.id for saved_query in unread])
+ unread_ids = [saved_query.id for saved_query in unread]
+ dependents = dependent_saved_query_ids(context.team_id, unread_ids)
+ feeding_consumers = saved_queries_feeding_consumers(context.team_id, unread_ids)
drafts = []
rejections = []
for saved_query in unread:
subject = _subject(saved_query)
- reason = self._rejection(context, saved_query, has_dependents=bool(dependents.get(saved_query.id)))
+ reason = self._rejection(
+ context,
+ saved_query,
+ has_dependents=bool(dependents.get(saved_query.id)),
+ feeds_consumers=saved_query.id in feeding_consumers,
+ )
if reason is not None:
rejections.append(Rejection(subject=subject, reason=reason))
continue
@@ -75,12 +83,19 @@ class DeprecateCandidate(Candidate):
return saved_query is None or not saved_query.materializes or _is_read(context, subject)
def _rejection(
- self, context: CandidateContext, saved_query: SavedQueryDefinition, *, has_dependents: bool
+ self,
+ context: CandidateContext,
+ saved_query: SavedQueryDefinition,
+ *,
+ has_dependents: bool,
+ feeds_consumers: bool,
) -> str | None:
if saved_query.created_at > context.reads.window.starts_at:
return "created after the read window started"
if has_dependents:
return "other views read from it"
+ if feeds_consumers:
+ return "its refreshes feed person or account properties, a destination or a workflow"
if self._reads_direct_source(context, saved_query.id):
return "reads a direct-connection source, whose reads are not logged"
return None
diff --git a/products/warehouse_suggestions/backend/presentation/serializers.py b/products/warehouse_suggestions/backend/presentation/serializers.py
index 69acb1cf48ee..00513ce1111a 100644
--- a/products/warehouse_suggestions/backend/presentation/serializers.py
+++ b/products/warehouse_suggestions/backend/presentation/serializers.py
@@ -303,4 +303,10 @@ class AcceptWarehouseSuggestionSerializer(DataclassSerializer):
"incremental_key value each refresh reads again, in seconds, so late rows are picked up. Omit to "
"keep the proposed lookback.",
},
+ "keep_materialized": {
+ "allow_null": False,
+ "help_text": "Deprecate only: true to keep a materialized view refreshing after it is deprecated. "
+ "By default, accepting a deprecate suggestion also stops the view's refreshes and deletes its "
+ "stored table, unless other views or syncs depend on it.",
+ },
}
diff --git a/products/warehouse_suggestions/backend/tests/test_accept.py b/products/warehouse_suggestions/backend/tests/test_accept.py
index 7cd5b25ce4ed..63a1e30dc009 100644
--- a/products/warehouse_suggestions/backend/tests/test_accept.py
+++ b/products/warehouse_suggestions/backend/tests/test_accept.py
@@ -1,4 +1,6 @@
+from collections.abc import Callable
from dataclasses import replace
+from datetime import timedelta
from uuid import UUID
from posthog.test.base import APIBaseTest, NonAtomicAPIBaseTest
@@ -12,9 +14,13 @@ from posthog.models.activity_logging.activity_log import ActivityLog
from posthog.models.personal_api_key import PersonalAPIKey
from posthog.models.utils import generate_random_token_personal, hash_key_value
+from products.cdp.backend.models.hog_functions import HogFunction
from products.data_catalog.backend.facade.api import certifications_for_team, propose_certification
from products.data_catalog.backend.facade.enums import CertificationStatus
-from products.data_modeling.backend.facade.models import DAG, DataWarehouseSavedQuery, Node, NodeType
+from products.data_modeling.backend.facade.api import get_declared_target, set_declared_target
+from products.data_modeling.backend.facade.models import DAG, DataWarehouseSavedQuery, Edge, Node, NodeType
+from products.data_tools.backend.facade.models import DataWarehouseJoin
+from products.warehouse_sources.backend.facade.models import DataWarehouseTable
from products.warehouse_suggestions.backend.facade.contracts import (
CertifyPayload,
DeprecatePayload,
@@ -86,6 +92,33 @@ def create_view_in_dag(team: Team, user: User) -> DataWarehouseSavedQuery:
return view
+def materialize_in_place(view: DataWarehouseSavedQuery) -> DataWarehouseTable:
+ table = DataWarehouseTable.objects.create(team=view.team, name=view.name, format="Delta")
+ DataWarehouseSavedQuery.objects.filter(id=view.id).update(is_materialized=True, table=table)
+ Node.objects.filter(saved_query=view).update(type=NodeType.MAT_VIEW)
+ set_declared_target(Node.objects.get(saved_query=view), timedelta(hours=6))
+ return table
+
+
+def add_dependent_view(view: DataWarehouseSavedQuery) -> None:
+ node = Node.objects.get(saved_query=view)
+ dependent = DataWarehouseSavedQuery.objects.create(
+ team=view.team, name="orders_daily", query={"kind": "HogQLQuery", "query": "SELECT * FROM orders"}
+ )
+ dependent_node = Node.objects.create(
+ team=view.team, dag=node.dag, name="orders_daily", saved_query=dependent, type=NodeType.VIEW
+ )
+ Edge.objects.create(team=view.team, dag=node.dag, source=node, target=dependent_node)
+
+
+def add_destination(view: DataWarehouseSavedQuery) -> None:
+ HogFunction.objects.create(
+ team=view.team,
+ enabled=True,
+ filters={"source": "data-warehouse-view", "data_warehouse": [{"table_name": view.name}]},
+ )
+
+
def suggest(team_id: int, view_id: UUID, kind: WarehouseSuggestionKind) -> WarehouseSuggestion:
draft = make_draft(fingerprint=f"{kind}:orders", subject_id=view_id)
return ingest_surfaced(team_id, replace(draft, kind=kind, payload=PAYLOADS[kind]))
@@ -176,7 +209,7 @@ class TestAcceptSuggestion(APIBaseTest):
accepted = self._accept(suggestion.id)
suggestion.refresh_from_db()
- reused_asset = {"certification_id": str(certification.id)}
+ reused_asset = {"certification_id": str(certification.id), "stopped_materialization": False}
assert accepted["http_status"] == expected_http_status, accepted
assert (suggestion.status, suggestion.created_asset, self._certification_status()) == (
expected_suggestion_status,
@@ -184,6 +217,104 @@ class TestAcceptSuggestion(APIBaseTest):
expected_certification_status,
)
+ @parameterized.expand(
+ [
+ ("stops_a_materialized_view", True, {}, None, True),
+ ("keeps_it_when_asked", True, {"keep_materialized": True}, None, False),
+ ("leaves_a_plain_view_alone", False, {}, None, False),
+ ("keeps_it_when_a_view_was_built_on_it", True, {}, add_dependent_view, False),
+ ("keeps_it_when_it_feeds_a_destination", True, {}, add_destination, False),
+ ]
+ )
+ def test_accepting_a_deprecate_suggestion_stops_refreshes_only_when_safe(
+ self,
+ _name: str,
+ materialized: bool,
+ body: dict,
+ add_reader: Callable[[DataWarehouseSavedQuery], None] | None,
+ expect_stopped: bool,
+ ) -> None:
+ table = materialize_in_place(self.view) if materialized else None
+ view_join = DataWarehouseJoin.objects.create(
+ team=self.team,
+ source_table_name=self.view.name,
+ source_table_key="distinct_id",
+ joining_table_name="events",
+ joining_table_key="distinct_id",
+ field_name="events",
+ )
+ draft = make_draft(fingerprint="deprecate:orders", subject_id=self.view.id)
+ payload = DeprecatePayload(subject_name="orders", refresh_seconds_per_month=900, refresh_bytes_per_month=10**9)
+ suggestion = ingest_surfaced(
+ self.team.id, replace(draft, kind=WarehouseSuggestionKind.DEPRECATE, payload=payload)
+ )
+ if add_reader is not None:
+ add_reader(self.view)
+
+ with patch(REPORT) as report:
+ accepted = self._accept(suggestion.id, body)
+
+ self.view.refresh_from_db()
+ assert accepted["http_status"] == status.HTTP_200_OK, accepted
+ assert self._certification_status() == CertificationStatus.DEPRECATED
+ assert accepted["created_asset"]["stopped_materialization"] is expect_stopped
+ assert (Node.objects.get(team=self.team, saved_query=self.view).type == NodeType.VIEW) is (
+ expect_stopped or not materialized
+ )
+ if table is not None:
+ table.refresh_from_db()
+ assert bool(table.deleted) is expect_stopped
+ view_join.refresh_from_db()
+ assert not view_join.deleted
+ disabled_trigger = (
+ ActivityLog.objects.filter(
+ team_id=self.team.id, item_id=str(self.view.id), activity="materialization_disabled"
+ )
+ .values_list("detail__trigger", flat=True)
+ .first()
+ )
+ assert disabled_trigger == (
+ {"job_type": "warehouse_suggestion", "job_id": str(suggestion.id), "payload": {"kind": "deprecate"}}
+ if expect_stopped
+ else None
+ )
+ properties = report.call_args.args[1]
+ assert (
+ properties["stopped_materialization"],
+ properties["refresh_seconds_per_month"],
+ properties["refresh_bytes_per_month"],
+ ) == ((True, 900, 10**9) if expect_stopped else (False, 0, 0))
+
+ @parameterized.expand(
+ [
+ ("node_type_update", "products.data_modeling.backend.logic.saved_query_materialization.update_node_type"),
+ (
+ "frequency_target_clear",
+ "products.data_modeling.backend.logic.schedule_reconcile.apply_saved_query_frequency_target",
+ ),
+ ]
+ )
+ def test_a_deprecate_whose_stop_fails_leaves_the_view_materialized_and_a_retry_stops_it(
+ self, _name: str, failing_step: str
+ ) -> None:
+ table = materialize_in_place(self.view)
+ suggestion = self._suggest(WarehouseSuggestionKind.DEPRECATE)
+
+ with patch(failing_step, side_effect=RuntimeError("database is down")):
+ failed = self._accept(suggestion.id)
+ suggestion.refresh_from_db()
+ self.view.refresh_from_db()
+ table.refresh_from_db()
+ untouched = (self.view.is_materialized, self.view.table_id, bool(table.deleted))
+ retried = self._accept(suggestion.id)
+
+ node = Node.objects.get(team=self.team, saved_query=self.view)
+ assert failed["http_status"] == status.HTTP_500_INTERNAL_SERVER_ERROR, failed
+ assert suggestion.status == WarehouseSuggestionStatus.PROPOSED
+ assert untouched == (True, table.id, False)
+ assert retried["created_asset"]["stopped_materialization"] is True, retried
+ assert (node.type, get_declared_target(node)) == (NodeType.VIEW, None)
+
def test_accepting_a_materialize_suggestion_materializes_the_view_on_the_chosen_interval(self) -> None:
suggestion = self._suggest(WarehouseSuggestionKind.MATERIALIZE)
@@ -315,6 +446,13 @@ class TestAcceptSuggestion(APIBaseTest):
status.HTTP_400_BAD_REQUEST,
"refresh_interval_seconds",
),
+ (
+ "keep_materialized_on_a_materialize_suggestion",
+ MATERIALIZE_PAYLOAD,
+ {"keep_materialized": True},
+ status.HTTP_400_BAD_REQUEST,
+ "keep_materialized",
+ ),
(
"lookback_only_on_stale_suggested_keys",
replace(INCREMENTAL_DAY_PAYLOAD, incremental_key="hour", unique_key=("hour",)),
diff --git a/products/warehouse_suggestions/backend/tests/test_candidates.py b/products/warehouse_suggestions/backend/tests/test_candidates.py
index e73d6c3283d3..a5dac5993129 100644
--- a/products/warehouse_suggestions/backend/tests/test_candidates.py
+++ b/products/warehouse_suggestions/backend/tests/test_candidates.py
@@ -3,6 +3,7 @@ from datetime import timedelta
from uuid import uuid4
from posthog.test.base import BaseTest
+from unittest.mock import patch
from django.test import SimpleTestCase
@@ -318,3 +319,17 @@ class TestDeprecateBackgroundReads(BaseTest):
result = DeprecateCandidate().evaluate(candidate_context)
assert [draft.subject_id for draft in result.drafts] == expected_subjects
+
+ def test_a_view_feeding_a_consumer_is_not_proposed(self) -> None:
+ candidate_context = context(team_reads({}), views=[view(VIEW_ID, materializes=True)], team_id=self.team.pk)
+
+ with patch(
+ "products.warehouse_suggestions.backend.logic.candidates.deprecate.saved_queries_feeding_consumers",
+ return_value=frozenset({VIEW_ID}),
+ ):
+ result = DeprecateCandidate().evaluate(candidate_context)
+
+ assert (result.drafts, [rejection.reason for rejection in result.rejections]) == (
+ (),
+ ["its refreshes feed person or account properties, a destination or a workflow"],
+ )
diff --git a/products/warehouse_suggestions/frontend/generated/api.schemas.ts b/products/warehouse_suggestions/frontend/generated/api.schemas.ts
index 326ae4d50de9..3907131e0b62 100644
--- a/products/warehouse_suggestions/frontend/generated/api.schemas.ts
+++ b/products/warehouse_suggestions/frontend/generated/api.schemas.ts
@@ -318,6 +318,8 @@ export interface AcceptWarehouseSuggestionApi {
* @maximum 2592000
*/
lookback_seconds?: number
+ /** Deprecate only: true to keep a materialized view refreshing after it is deprecated. By default, accepting a deprecate suggestion also stops the view's refreshes and deletes its stored table, unless other views or syncs depend on it. */
+ keep_materialized?: boolean
}
export interface DismissWarehouseSuggestionApi {
diff --git a/products/warehouse_suggestions/frontend/generated/api.zod.ts b/products/warehouse_suggestions/frontend/generated/api.zod.ts
index c44550bab68c..f613afd0349f 100644
--- a/products/warehouse_suggestions/frontend/generated/api.zod.ts
+++ b/products/warehouse_suggestions/frontend/generated/api.zod.ts
@@ -50,6 +50,12 @@ export const WarehouseSuggestionsAcceptCreateBody = /* @__PURE__ */ zod.object({
.describe(
"Materialize only, incremental refresh only: how far before the last refresh's highest incremental_key value each refresh reads again, in seconds, so late rows are picked up. Omit to keep the proposed lookback."
),
+ keep_materialized: zod
+ .boolean()
+ .optional()
+ .describe(
+ "Deprecate only: true to keep a materialized view refreshing after it is deprecated. By default, accepting a deprecate suggestion also stops the view's refreshes and deletes its stored table, unless other views or syncs depend on it."
+ ),
})
export const warehouseSuggestionsDismissCreateBodyNoteMax = 1000
diff --git a/products/warehouse_suggestions/mcp/tools.yaml b/products/warehouse_suggestions/mcp/tools.yaml
index 6658c788de50..e2e9f7e15f38 100644
--- a/products/warehouse_suggestions/mcp/tools.yaml
+++ b/products/warehouse_suggestions/mcp/tools.yaml
@@ -22,20 +22,24 @@ tools:
title: Accept a warehouse suggestion
description: >
Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as
- certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on
- the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another
- schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by
- incremental_key and update rows by unique_key, full refreshes rebuild the whole table. The payload is a
- starting point. To change it, pass any of refresh_interval_seconds, refresh_mode, incremental_key,
- unique_key and lookback_seconds; an omitted field keeps the suggested value. Check keys the person chose
- with view-check-incremental first. A 400 names the field the server refused and why. A 409 means the view
- changed since the suggestion was made and its suggested keys no longer apply. Requires human typed
- confirmation.
+ certified or deprecated in the data catalog. Accepting a deprecate suggestion on a materialized view also
+ stops its refreshes and deletes its stored table, so the view is computed when read; pass
+ keep_materialized=true to keep it refreshing. The view stays materialized when other views read from it or
+ its refreshes feed person properties, a destination or a workflow; `created_asset.stopped_materialization`
+ says whether refreshes stopped. Materialize suggestions turn on materialization for the view on the proposed
+ schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They
+ also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and
+ update rows by unique_key, full refreshes rebuild the whole table. The payload is a starting point. To
+ change it, pass any of refresh_interval_seconds, refresh_mode, incremental_key, unique_key and
+ lookback_seconds; an omitted field keeps the suggested value. Check keys the person chose with
+ view-check-incremental first. A 400 names the field the server refused and why. A 409 means the view changed
+ since the suggestion was made and its suggested keys no longer apply. Requires human typed confirmation.
confirmed_action:
message: >
About to accept warehouse suggestion '{id}'. This applies the change it proposes to the view or table.
If it is a materialize suggestion, the view then refreshes on a schedule, which adds a recurring compute
- cost. Reply 'confirm' to proceed.
+ cost. If it is a deprecate suggestion on a materialized view, its refreshes stop and its stored table is
+ deleted, unless keep_materialized is true. Reply 'confirm' to proceed.
action_label: accept warehouse suggestion
warehouse-suggestions-dismiss:
operation: warehouse_suggestions_dismiss_create
diff --git a/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md b/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md
index 7d5b309638f3..10cc5ba5a01e 100644
--- a/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md
+++ b/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md
@@ -36,7 +36,7 @@ Do not invent a score or a confidence level.
- `refresh_interval_seconds` is the proposed interval. `refresh_interval_basis` says what kept it from being shorter: `source_sync` (the sources sync no faster), `fastest_schedule` (PostHog refreshes views no more often), `read_rate` (people read the view no more often) or `savings` (a shorter interval costs more than it saves).
- `refresh_mode` is `incremental` when each refresh can read only new rows, by `incremental_key`, and update rows by `unique_key`. Otherwise it is `full`, and each refresh rebuilds the whole table.
- **certify**: many people read this view or table, from several places. Use the read counts in `evidence`.
-- **deprecate**: nobody reads this materialized view, but it still refreshes. `refresh_seconds_per_month` and `refresh_bytes_per_month` are that cost. Accepting only marks it deprecated in the data catalog. To stop the cost, offer `view-unmaterialize` as well.
+- **deprecate**: nobody reads this materialized view, but it still refreshes. `refresh_seconds_per_month` and `refresh_bytes_per_month` are that cost. Accepting marks it deprecated in the data catalog, stops its refreshes and deletes its stored table. The view definition stays, so it can be materialized again later. To keep it refreshing, pass `keep_materialized: true`. After accepting, `created_asset.stopped_materialization` says whether the refreshes stopped: they keep running when other views read from it or its refreshes feed person properties, a destination or a workflow.
## 3. Change a materialize suggestion before accepting
diff --git a/products/workflows/backend/facade/api.py b/products/workflows/backend/facade/api.py
index a513ba552bcc..76fc797a34f0 100644
--- a/products/workflows/backend/facade/api.py
+++ b/products/workflows/backend/facade/api.py
@@ -237,6 +237,20 @@ def has_active_workflow_for_warehouse_table(*, team_id: int, trigger_source: str
).exists()
+def active_workflow_warehouse_table_names(
+ *, team_id: int, trigger_source: str, table_names: Collection[str]
+) -> set[str]:
+ """The given warehouse table names that an active workflow triggers on."""
+ return set(
+ HogFlow.objects.filter(
+ team_id=team_id,
+ status=HogFlow.State.ACTIVE,
+ trigger__type=trigger_source,
+ trigger__table_name__in=list(table_names),
+ ).values_list("trigger__table_name", flat=True)
+ )
+
+
def get_workflow_activity_summary(*, team_id: int, recent_limit: int) -> WorkflowActivitySummary:
"""Total and non-archived workflow counts, plus the most recently updated workflows."""
qs = HogFlow.objects.filter(team_id=team_id)
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index 58df8e167c9a..62d07ad723fe 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -15854,7 +15854,7 @@
}
},
"warehouse-suggestions-accept-execute": {
- "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. The payload is a starting point. To change it, pass any of refresh_interval_seconds, refresh_mode, incremental_key, unique_key and lookback_seconds; an omitted field keeps the suggested value. Check keys the person chose with view-check-incremental first. A 400 names the field the server refused and why. A 409 means the view changed since the suggestion was made and its suggested keys no longer apply. Requires human typed confirmation.",
+ "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Accepting a deprecate suggestion on a materialized view also stops its refreshes and deletes its stored table, so the view is computed when read; pass keep_materialized=true to keep it refreshing. The view stays materialized when other views read from it or its refreshes feed person properties, a destination or a workflow; `created_asset.stopped_materialization` says whether refreshes stopped. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. The payload is a starting point. To change it, pass any of refresh_interval_seconds, refresh_mode, incremental_key, unique_key and lookback_seconds; an omitted field keeps the suggested value. Check keys the person chose with view-check-incremental first. A 400 names the field the server refused and why. A 409 means the view changed since the suggestion was made and its suggested keys no longer apply. Requires human typed confirmation.",
"category": "Warehouse suggestions",
"feature": "warehouse_suggestions",
"summary": "Accept a warehouse suggestion (execute)",
@@ -15870,7 +15870,7 @@
"feature_flag": "warehouse-suggestions"
},
"warehouse-suggestions-accept-prepare": {
- "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. The payload is a starting point. To change it, pass any of refresh_interval_seconds, refresh_mode, incremental_key, unique_key and lookback_seconds; an omitted field keeps the suggested value. Check keys the person chose with view-check-incremental first. A 400 names the field the server refused and why. A 409 means the view changed since the suggestion was made and its suggested keys no longer apply. Requires human typed confirmation.",
+ "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Accepting a deprecate suggestion on a materialized view also stops its refreshes and deletes its stored table, so the view is computed when read; pass keep_materialized=true to keep it refreshing. The view stays materialized when other views read from it or its refreshes feed person properties, a destination or a workflow; `created_asset.stopped_materialization` says whether refreshes stopped. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. The payload is a starting point. To change it, pass any of refresh_interval_seconds, refresh_mode, incremental_key, unique_key and lookback_seconds; an omitted field keeps the suggested value. Check keys the person chose with view-check-incremental first. A 400 names the field the server refused and why. A 409 means the view changed since the suggestion was made and its suggested keys no longer apply. Requires human typed confirmation.",
"category": "Warehouse suggestions",
"feature": "warehouse_suggestions",
"summary": "Accept a warehouse suggestion (prepare)",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index 136e7ca88600..e43702b29c29 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -16600,7 +16600,7 @@
}
},
"warehouse-suggestions-accept-execute": {
- "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. The payload is a starting point. To change it, pass any of refresh_interval_seconds, refresh_mode, incremental_key, unique_key and lookback_seconds; an omitted field keeps the suggested value. Check keys the person chose with view-check-incremental first. A 400 names the field the server refused and why. A 409 means the view changed since the suggestion was made and its suggested keys no longer apply. Requires human typed confirmation.",
+ "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Accepting a deprecate suggestion on a materialized view also stops its refreshes and deletes its stored table, so the view is computed when read; pass keep_materialized=true to keep it refreshing. The view stays materialized when other views read from it or its refreshes feed person properties, a destination or a workflow; `created_asset.stopped_materialization` says whether refreshes stopped. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. The payload is a starting point. To change it, pass any of refresh_interval_seconds, refresh_mode, incremental_key, unique_key and lookback_seconds; an omitted field keeps the suggested value. Check keys the person chose with view-check-incremental first. A 400 names the field the server refused and why. A 409 means the view changed since the suggestion was made and its suggested keys no longer apply. Requires human typed confirmation.",
"category": "Warehouse suggestions",
"feature": "warehouse_suggestions",
"summary": "Accept a warehouse suggestion (execute)",
@@ -16616,7 +16616,7 @@
"feature_flag": "warehouse-suggestions"
},
"warehouse-suggestions-accept-prepare": {
- "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. The payload is a starting point. To change it, pass any of refresh_interval_seconds, refresh_mode, incremental_key, unique_key and lookback_seconds; an omitted field keeps the suggested value. Check keys the person chose with view-check-incremental first. A 400 names the field the server refused and why. A 409 means the view changed since the suggestion was made and its suggested keys no longer apply. Requires human typed confirmation.",
+ "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Accepting a deprecate suggestion on a materialized view also stops its refreshes and deletes its stored table, so the view is computed when read; pass keep_materialized=true to keep it refreshing. The view stays materialized when other views read from it or its refreshes feed person properties, a destination or a workflow; `created_asset.stopped_materialization` says whether refreshes stopped. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. The payload is a starting point. To change it, pass any of refresh_interval_seconds, refresh_mode, incremental_key, unique_key and lookback_seconds; an omitted field keeps the suggested value. Check keys the person chose with view-check-incremental first. A 400 names the field the server refused and why. A 409 means the view changed since the suggestion was made and its suggested keys no longer apply. Requires human typed confirmation.",
"category": "Warehouse suggestions",
"feature": "warehouse_suggestions",
"summary": "Accept a warehouse suggestion (prepare)",
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index cdd9fed274b7..4dc1ea767387 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -375,6 +375,8 @@ export namespace Schemas {
* @maximum 2592000
*/
lookback_seconds?: number;
+ /** Deprecate only: true to keep a materialized view refreshing after it is deprecated. By default, accepting a deprecate suggestion also stops the view's refreshes and deletes its stored table, unless other views or syncs depend on it. */
+ keep_materialized?: boolean;
}
export interface AccessControlResourceDefault {
diff --git a/services/mcp/src/generated/warehouse_suggestions/api.ts b/services/mcp/src/generated/warehouse_suggestions/api.ts
index 111e955b591b..88eb6b6b64b0 100644
--- a/services/mcp/src/generated/warehouse_suggestions/api.ts
+++ b/services/mcp/src/generated/warehouse_suggestions/api.ts
@@ -101,6 +101,12 @@ export const WarehouseSuggestionsAcceptCreateBody = () => zod.object({
.describe(
"Materialize only, incremental refresh only: how far before the last refresh's highest incremental_key value each refresh reads again, in seconds, so late rows are picked up. Omit to keep the proposed lookback."
),
+ keep_materialized: zod
+ .boolean()
+ .optional()
+ .describe(
+ "Deprecate only: true to keep a materialized view refreshing after it is deprecated. By default, accepting a deprecate suggestion also stops the view's refreshes and deletes its stored table, unless other views or syncs depend on it."
+ ),
})
export const warehouseSuggestionsDismissCreatePathIdRegExp = new RegExp(
diff --git a/services/mcp/src/tools/generated/warehouse_suggestions.ts b/services/mcp/src/tools/generated/warehouse_suggestions.ts
index eeb9966c8dc4..7ba84ba196cc 100644
--- a/services/mcp/src/tools/generated/warehouse_suggestions.ts
+++ b/services/mcp/src/tools/generated/warehouse_suggestions.ts
@@ -41,7 +41,7 @@ const warehouseSuggestionsAcceptPrepare = (): ToolBase<
purpose: 'warehouse-suggestions-accept',
actionLabel: 'accept warehouse suggestion',
messageTemplate:
- "About to accept warehouse suggestion '{id}'. This applies the change it proposes to the view or table. If it is a materialize suggestion, the view then refreshes on a schedule, which adds a recurring compute cost. Reply 'confirm' to proceed.\n",
+ "About to accept warehouse suggestion '{id}'. This applies the change it proposes to the view or table. If it is a materialize suggestion, the view then refreshes on a schedule, which adds a recurring compute cost. If it is a deprecate suggestion on a materialized view, its refreshes stop and its stored table is deleted, unless keep_materialized is true. Reply 'confirm' to proceed.\n",
codec: __runtime.codec,
stash: __runtime.stash,
boundScope: { projectId: String(__scopeProjectId) },
@@ -90,6 +90,9 @@ const warehouseSuggestionsAcceptExecute = (): ToolBase<
if (params.lookback_seconds !== undefined) {
body['lookback_seconds'] = params.lookback_seconds
}
+ if (params.keep_materialized !== undefined) {
+ body['keep_materialized'] = params.keep_materialized
+ }
const result = await context.api.request<Schemas.WarehouseSuggestion>({
method: 'POST',
path: `/api/projects/${encodeURIComponent(String(projectId))}/warehouse_suggestions/${encodeURIComponent(String(params.id))}/accept/`,
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/warehouse-suggestions-accept-prepare.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/warehouse-suggestions-accept-prepare.json
index a43944afc0b6..477b0a7f3521 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/warehouse-suggestions-accept-prepare.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/warehouse-suggestions-accept-prepare.json
@@ -9,6 +9,10 @@
"description": "Materialize only, incremental refresh only: an output column whose value only grows as rows arrive, such as a date, a datetime or an increasing number, instead of the proposed one. view-check-incremental lists the columns that qualify. Omit to keep the proposed key.",
"type": "string"
},
+ "keep_materialized": {
+ "description": "Deprecate only: true to keep a materialized view refreshing after it is deprecated. By default, accepting a deprecate suggestion also stops the view's refreshes and deletes its stored table, unless other views or syncs depend on it.",
+ "type": "boolean"
+ },
"lookback_seconds": {
"description": "Materialize only, incremental refresh only: how far before the last refresh's highest incremental_key value each refresh reads again, in seconds, so late rows are picked up. Omit to keep the proposed lookback.",
"maximum": 2592000,