:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-suggestions): propose the refresh mode and freshness (#119703)
Thiago Salvatore593fa0f6bb3e
diff --git a/products/warehouse_suggestions/backend/facade/contracts.py b/products/warehouse_suggestions/backend/facade/contracts.pyindex a43224782d4b..158e987d22af 100644--- a/products/warehouse_suggestions/backend/facade/contracts.py+++ b/products/warehouse_suggestions/backend/facade/contracts.py@@ -10,6 +10,8 @@ from .enums import (WarehouseSuggestionAssetOutcome,WarehouseSuggestionDismissalReason,WarehouseSuggestionKind,+ WarehouseSuggestionRefreshIntervalBasis,+ WarehouseSuggestionRefreshMode,WarehouseSuggestionStatus,WarehouseSuggestionSubjectKind,)@@ -53,6 +55,11 @@ class MaterializePayload:freshness_after_seconds: intlive_sources: tuple[SourceRef, ...]unknown_sources: tuple[SourceRef, ...]+ refresh_mode: WarehouseSuggestionRefreshMode = WarehouseSuggestionRefreshMode.FULL+ incremental_key: str | None = None+ unique_key: tuple[str, ...] = ()+ lookback_seconds: int = 0+ refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasis | None = NoneSuggestionPayload = CertifyPayload | DeprecatePayload | MaterializePayload@@ -76,6 +83,11 @@ class MaterializeSuggestionPayload:freshness_after_seconds: intlive_sources: VisibleSourcesunknown_sources: VisibleSources+ refresh_mode: WarehouseSuggestionRefreshMode = WarehouseSuggestionRefreshMode.FULL+ incremental_key: str | None = None+ unique_key: tuple[str, ...] = ()+ lookback_seconds: int = 0+ refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasis | None = NoneSuggestionPayloadView = CertifyPayload | DeprecatePayload | MaterializeSuggestionPayload@@ -198,5 +210,9 @@ class RefreshIntervalRefusedError(Exception):pass+class SuggestionOutdatedError(Exception):+ pass++class AcceptFailedError(Exception):passdiff --git a/products/warehouse_suggestions/backend/facade/enums.py b/products/warehouse_suggestions/backend/facade/enums.pyindex 3c1529d2b104..c449d9d0ba05 100644--- a/products/warehouse_suggestions/backend/facade/enums.py+++ b/products/warehouse_suggestions/backend/facade/enums.py@@ -32,3 +32,15 @@ class WarehouseSuggestionAssetOutcome(LabeledStrEnum):LIVE = "live"DELETED = "deleted"UNUSED = "unused"+++class WarehouseSuggestionRefreshMode(LabeledStrEnum):+ FULL = "full"+ INCREMENTAL = "incremental"+++class WarehouseSuggestionRefreshIntervalBasis(LabeledStrEnum):+ SOURCE_SYNC = ("source_sync", "Source sync")+ FASTEST_SCHEDULE = ("fastest_schedule", "Fastest schedule")+ READ_RATE = ("read_rate", "Read rate")+ SAVINGS = "savings"diff --git a/products/warehouse_suggestions/backend/logic/accept.py b/products/warehouse_suggestions/backend/logic/accept.pyindex adb971dedca2..39ab2b1aeb27 100644--- a/products/warehouse_suggestions/backend/logic/accept.py+++ b/products/warehouse_suggestions/backend/logic/accept.py@@ -10,12 +10,18 @@ from posthog.dataclasses import frozenfrom products.data_catalog.backend.facade.api import certifications_for_team, certify, deprecate, propose_certificationfrom products.data_catalog.backend.facade.enums import CertificationStatusfrom products.data_modeling.backend.facade.api import (+ IncrementalConfig,+ IncrementalConfigRefusedError,MaterializationFailedError,MaterializationForbiddenError,MaterializationRefusedError,SavedQueryNotFoundError,+ definition_database,enable_saved_query_materialization,+ get_saved_query_columns,+ get_saved_query_sql,get_saved_query_summary,+ propose_incremental_config,)from products.warehouse_sources.backend.facade.api import get_queryable_table@@ -29,10 +35,16 @@ from ..facade.contracts import (SubjectAlreadyCertifiedError,SubjectEditAccessRequiredError,SuggestionAlreadyDecidedError,+ SuggestionOutdatedError,SuggestionPayload,SuggestionSubjectGoneError,)-from ..facade.enums import WarehouseSuggestionKind, WarehouseSuggestionStatus, WarehouseSuggestionSubjectKind+from ..facade.enums import (+ WarehouseSuggestionKind,+ WarehouseSuggestionRefreshMode,+ WarehouseSuggestionStatus,+ WarehouseSuggestionSubjectKind,+)from ..models import WarehouseSuggestionfrom .analytics import SuggestionOutcome, report_outcomesfrom .payloads import payload_from_json@@ -94,6 +106,11 @@ class CertificationAcceptor(Acceptor):return existing+VIEW_CHANGED_SINCE_SUGGESTED = (+ "This view changed after the suggestion was made, so it can no longer refresh incrementally on the suggested keys."+)++class MaterializeAcceptor(Acceptor):def accept(self, suggestion: WarehouseSuggestion, payload: SuggestionPayload, request: AcceptRequest@@ -101,6 +118,9 @@ class MaterializeAcceptor(Acceptor):if not isinstance(payload, MaterializePayload):raise AcceptFailedError("This suggestion has no refresh interval to apply.")interval = request.refresh_interval or timedelta(seconds=payload.refresh_interval_seconds)+ incremental_config = _incremental_config(payload)+ if incremental_config is not None and not _still_proposed(suggestion, incremental_config, request.user):+ raise SuggestionOutdatedError(VIEW_CHANGED_SINCE_SUGGESTED)try:enable_saved_query_materialization(suggestion.team_id,@@ -109,11 +129,14 @@ class MaterializeAcceptor(Acceptor):sync_frequency_interval=interval,visible_blocker_names=_no_blocker_names,was_impersonated=request.was_impersonated,+ incremental_config=incremental_config,)except SavedQueryNotFoundError:raise SuggestionSubjectGoneError(WarehouseSuggestionSubjectKind.SAVED_QUERY)except MaterializationForbiddenError:raise SubjectEditAccessRequiredError(WarehouseSuggestionSubjectKind.SAVED_QUERY)+ except IncrementalConfigRefusedError as error:+ raise SuggestionOutdatedError(f"{VIEW_CHANGED_SINCE_SUGGESTED} {error}")except MaterializationRefusedError as error:raise RefreshIntervalRefusedError(str(error))except MaterializationFailedError as error:@@ -123,6 +146,29 @@ class MaterializeAcceptor(Acceptor):)+def _incremental_config(payload: MaterializePayload) -> IncrementalConfig | None:+ if payload.refresh_mode != WarehouseSuggestionRefreshMode.INCREMENTAL or payload.incremental_key is None:+ return None+ return IncrementalConfig(+ incremental_key=payload.incremental_key,+ unique_key=payload.unique_key,+ lookback_seconds=payload.lookback_seconds,+ )+++def _still_proposed(suggestion: WarehouseSuggestion, config: IncrementalConfig, user: "User") -> bool:+ proposed = propose_incremental_config(+ get_saved_query_sql(suggestion.team_id, suggestion.subject_id) or "",+ get_saved_query_columns(suggestion.team_id, suggestion.subject_id),+ database=definition_database(suggestion.team_id, user=user),+ )+ return (+ proposed is not None+ and proposed.incremental_key == config.incremental_key+ and set(proposed.unique_key) == set(config.unique_key)+ )++ACCEPTORS: Mapping[WarehouseSuggestionKind, Acceptor] = {WarehouseSuggestionKind.CERTIFY: CertificationAcceptor(status=CertificationStatus.CERTIFIED, apply=certify, reuses_existing=Falsediff --git a/products/warehouse_suggestions/backend/logic/candidates/materialize.py b/products/warehouse_suggestions/backend/logic/candidates/materialize.pyindex 8fa07332736b..a409a9ef5190 100644--- a/products/warehouse_suggestions/backend/logic/candidates/materialize.py+++ b/products/warehouse_suggestions/backend/logic/candidates/materialize.py@@ -1,21 +1,35 @@from collections.abc import Iterable+from dataclasses import replacefrom datetime import datetime, timedelta+from functools import cached_propertyfrom uuid import UUID+from posthog.hogql.database.database import Database+from posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exceptionfrom products.data_modeling.backend.facade.api import (+ IncrementalConfig,MissingDagNodeError,SavedQueryFrequencyBounds,- check_incremental_eligibility,+ definition_database,+ get_saved_query_columns,+ incremental_enabled,+ propose_incremental_config,saved_query_target_bounds,upstream_table_refs,)from products.data_modeling.backend.facade.contracts import SavedQueryDefinitionfrom ...facade.contracts import MaterializePayload, SourceRef-from ...facade.enums import WarehouseSuggestionAssetOutcome, WarehouseSuggestionKind, WarehouseSuggestionSubjectKind+from ...facade.enums import (+ WarehouseSuggestionAssetOutcome,+ WarehouseSuggestionKind,+ WarehouseSuggestionRefreshIntervalBasis,+ WarehouseSuggestionRefreshMode,+ WarehouseSuggestionSubjectKind,+)from ..reads import Subject, SubjectReadsfrom .base import DAYS_PER_MONTH, MILLISECONDS_PER_SECOND, Candidate, CandidateContext, CandidateResult, Rejection@@ -43,18 +57,44 @@ class _Proposal:estimate: SavingsEstimate+class _IncrementalProposer:+ """Proposes incremental keys for one team run, reading the flag and building the team database at most once."""++ def __init__(self, team_id: int) -> None:+ self._team_id = team_id++ @cached_property+ def _enabled(self) -> bool:+ return incremental_enabled(self._team_id)++ @cached_property+ def _database(self) -> Database | None:+ try:+ return definition_database(self._team_id)+ except Exception as error:+ capture_exception(error)+ return None++ def config_for(self, saved_query: SavedQueryDefinition) -> IncrementalConfig | None:+ if not self._enabled or self._database is None:+ return None+ column_types = get_saved_query_columns(self._team_id, saved_query.id)+ return propose_incremental_config(saved_query.hogql, column_types, database=self._database)++class MaterializeCandidate(Candidate):kind = WarehouseSuggestionKind.MATERIALIZEdef evaluate(self, context: CandidateContext) -> CandidateResult:drafts = []rejections = []+ incremental = _IncrementalProposer(context.team_id)for saved_query in context.inventory.saved_queries.values():subject = Subject(kind=WarehouseSuggestionSubjectKind.SAVED_QUERY, id=saved_query.id)reads = context.reads.reads_of(subject)if reads is None or reads.human_requests == 0 or not self._is_unmaterialized(context, saved_query):continue- outcome = self._propose(context, saved_query, reads)+ outcome = self._propose(context, saved_query, reads, incremental)if isinstance(outcome, str):rejections.append(Rejection(subject=subject, reason=outcome))continue@@ -92,15 +132,15 @@ class MaterializeCandidate(Candidate):return not saved_query.materializes and context.is_suggestible_view(saved_query)def _propose(- self, context: CandidateContext, saved_query: SavedQueryDefinition, reads: SubjectReads+ self,+ context: CandidateContext,+ saved_query: SavedQueryDefinition,+ reads: SubjectReads,+ incremental: _IncrementalProposer,) -> _Proposal | str:rejection = self._read_floor_rejection(context, reads)if rejection is not None:return rejection- if context.rules.materialize.require_incremental_eligibility and not _can_refresh_incrementally(- saved_query.hogql- ):- return "its query cannot refresh incrementally"frequency = saved_query_target_bounds(context.team_id, saved_query.id)if frequency is None:capture_exception(MissingDagNodeError(f"Saved query {saved_query.id} has no DAG node"))@@ -121,16 +161,25 @@ class MaterializeCandidate(Candidate):sources = _classify_sources(context, saved_query.id, frequency)freshness_today = int(frequency.max_data_age.total_seconds()) or Noneinterval_seconds = int(estimate.interval.total_seconds())+ payload = MaterializePayload(+ subject_name=saved_query.name,+ refresh_interval_seconds=interval_seconds,+ saves_seconds_per_month=estimate.seconds_saved,+ saves_bytes_per_month=estimate.bytes_saved,+ freshness_today_seconds=freshness_today,+ freshness_after_seconds=interval_seconds + (freshness_today or 0),+ live_sources=sources.live,+ unknown_sources=sources.unknown,+ refresh_interval_basis=interval_basis(+ estimate.interval,+ frequency.bounds.allowed,+ intervals,+ limited_by_sources=frequency.bounds.floor is not None,+ ),+ )return _Proposal(- payload=MaterializePayload(- subject_name=saved_query.name,- refresh_interval_seconds=interval_seconds,- saves_seconds_per_month=estimate.seconds_saved,- saves_bytes_per_month=estimate.bytes_saved,- freshness_today_seconds=freshness_today,- freshness_after_seconds=interval_seconds + (freshness_today or 0),- live_sources=sources.live,- unknown_sources=sources.unknown,+ payload=_with_refresh_mode(+ payload, incremental.config_for(saved_query), context.rules.materialize.incremental_lookback_seconds),estimate=estimate,)@@ -150,18 +199,43 @@ class MaterializeCandidate(Candidate):return None-def _can_refresh_incrementally(hogql: str) -> bool:- eligibility = check_incremental_eligibility(hogql, None)- return eligibility.eligible and bool(eligibility.key_candidates) and bool(eligibility.unique_key_candidates)--def allowed_intervals(context: CandidateContext, allowed: Iterable[timedelta], reads: SubjectReads) -> list[timedelta]:refresh = context.rules.refreshintervals = sorted(interval for interval in allowed if interval <= refresh.max_interval)read_day_share = reads.human_days / context.days_observedif read_day_share < refresh.min_read_day_share_below_max_interval:return [interval for interval in intervals if interval >= refresh.max_interval]- return intervals+ return [interval for interval in intervals if interval >= read_interval(reads)]+++def read_interval(reads: SubjectReads) -> timedelta:+ return timedelta(days=reads.human_days) / max(reads.human_requests, 1)+++def interval_basis(+ interval: timedelta, allowed: Iterable[timedelta], read_allowed: list[timedelta], *, limited_by_sources: bool+) -> WarehouseSuggestionRefreshIntervalBasis:+ if interval == min(allowed):+ if limited_by_sources:+ return WarehouseSuggestionRefreshIntervalBasis.SOURCE_SYNC+ return WarehouseSuggestionRefreshIntervalBasis.FASTEST_SCHEDULE+ if interval == read_allowed[0]:+ return WarehouseSuggestionRefreshIntervalBasis.READ_RATE+ return WarehouseSuggestionRefreshIntervalBasis.SAVINGS+++def _with_refresh_mode(+ payload: MaterializePayload, config: IncrementalConfig | None, lookback_seconds: int+) -> MaterializePayload:+ if config is None:+ return payload+ return replace(+ payload,+ refresh_mode=WarehouseSuggestionRefreshMode.INCREMENTAL,+ incremental_key=config.incremental_key,+ unique_key=config.unique_key,+ lookback_seconds=lookback_seconds,+ )def estimate_savings(context: CandidateContext, reads: SubjectReads, interval: timedelta) -> SavingsEstimate:diff --git a/products/warehouse_suggestions/backend/logic/payloads.py b/products/warehouse_suggestions/backend/logic/payloads.pyindex a759816541ea..fa2f8a617c29 100644--- a/products/warehouse_suggestions/backend/logic/payloads.py+++ b/products/warehouse_suggestions/backend/logic/payloads.py@@ -1,6 +1,6 @@import jsonfrom collections.abc import Collection, Iterable, Mapping-from dataclasses import asdict+from dataclasses import asdict, fieldsfrom typing import Anyfrom uuid import UUID@@ -16,7 +16,11 @@ from ..facade.contracts import (UnsupportedPayloadVersionError,VisibleSources,)-from ..facade.enums import WarehouseSuggestionKind+from ..facade.enums import (+ WarehouseSuggestionKind,+ WarehouseSuggestionRefreshIntervalBasis,+ WarehouseSuggestionRefreshMode,+)SOURCE_FIELDS = ("live_sources", "unknown_sources")@@ -35,7 +39,7 @@ def payload_from_json(if kind == WarehouseSuggestionKind.DEPRECATE:return DeprecatePayload(**data)sources = {field: tuple(_source_ref(source) for source in data[field]) for field in SOURCE_FIELDS}- return MaterializePayload(**{**data, **sources})+ return MaterializePayload(**{**data, **sources, **_refresh_fields(data)})def source_table_ids(payloads: Iterable[SuggestionPayload]) -> set[UUID]:@@ -51,13 +55,9 @@ def source_table_ids(payloads: Iterable[SuggestionPayload]) -> set[UUID]:def payload_view(payload: SuggestionPayload, readable_table_ids: Collection[UUID]) -> SuggestionPayloadView:if not isinstance(payload, MaterializePayload):return payload+ shared = {field.name: getattr(payload, field.name) for field in fields(payload) if field.name not in SOURCE_FIELDS}return MaterializeSuggestionPayload(- subject_name=payload.subject_name,- refresh_interval_seconds=payload.refresh_interval_seconds,- saves_seconds_per_month=payload.saves_seconds_per_month,- saves_bytes_per_month=payload.saves_bytes_per_month,- freshness_today_seconds=payload.freshness_today_seconds,- freshness_after_seconds=payload.freshness_after_seconds,+ **shared,live_sources=_visible_sources(payload.live_sources, readable_table_ids),unknown_sources=_visible_sources(payload.unknown_sources, readable_table_ids),)@@ -75,3 +75,12 @@ def _visible_sources(sources: tuple[SourceRef, ...], readable_table_ids: Collectdef _source_ref(source: Mapping[str, Any]) -> SourceRef:table_id = source["warehouse_table_id"]return SourceRef(name=source["name"], warehouse_table_id=UUID(table_id) if table_id else None)+++def _refresh_fields(data: Mapping[str, Any]) -> dict[str, Any]:+ basis = data.get("refresh_interval_basis")+ return {+ "refresh_mode": WarehouseSuggestionRefreshMode(data.get("refresh_mode", WarehouseSuggestionRefreshMode.FULL)),+ "unique_key": tuple(data.get("unique_key", ())),+ "refresh_interval_basis": WarehouseSuggestionRefreshIntervalBasis(basis) if basis else None,+ }diff --git a/products/warehouse_suggestions/backend/logic/rules.py b/products/warehouse_suggestions/backend/logic/rules.pyindex c6c38c978ab0..9a110781ec9c 100644--- a/products/warehouse_suggestions/backend/logic/rules.py+++ b/products/warehouse_suggestions/backend/logic/rules.py@@ -102,7 +102,7 @@ class MaterializeRules:min_seconds_saved: floatmin_bytes_saved: floatmin_saved_share: float- require_incremental_eligibility: bool+ incremental_lookback_seconds: int@frozen@@ -201,7 +201,7 @@ RULES = Rules(min_seconds_saved=600.0,min_bytes_saved=float(TERABYTE),min_saved_share=0.5,- require_incremental_eligibility=True,+ incremental_lookback_seconds=int(timedelta(days=3).total_seconds()),),refresh=RefreshRules(max_interval=timedelta(hours=24), min_read_day_share_below_max_interval=0.5),lifecycle=LifecycleRules(diff --git a/products/warehouse_suggestions/backend/presentation/serializers.py b/products/warehouse_suggestions/backend/presentation/serializers.pyindex 44e0c41679d9..95ed4f34c2e6 100644--- a/products/warehouse_suggestions/backend/presentation/serializers.py+++ b/products/warehouse_suggestions/backend/presentation/serializers.py@@ -21,6 +21,8 @@ from ..facade.enums import (WarehouseSuggestionAssetOutcome,WarehouseSuggestionDismissalReason,WarehouseSuggestionKind,+ WarehouseSuggestionRefreshIntervalBasis,+ WarehouseSuggestionRefreshMode,WarehouseSuggestionStatus,WarehouseSuggestionSubjectKind,)@@ -80,6 +82,20 @@ class MaterializePayloadSerializer(DataclassSerializer):unknown_sources = VisibleSourcesSerializer(help_text="Sources with no sync schedule, so their freshness is unknown.")+ refresh_mode = serializers.ChoiceField(+ choices=WarehouseSuggestionRefreshMode.choices,+ help_text="incremental: each refresh reads only rows with a new incremental_key value and updates rows by "+ "unique_key. full: each refresh rebuilds the whole table. Incremental is proposed only when the keys are "+ "certain from the query.",+ )+ refresh_interval_basis = serializers.ChoiceField(+ choices=WarehouseSuggestionRefreshIntervalBasis.choices,+ allow_null=True,+ help_text="Why the interval is not shorter. source_sync: the view's sources sync no faster. "+ "fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. "+ "savings: a shorter interval would cost more than it saves. Null on suggestions made before this was "+ "recorded.",+ )class Meta:dataclass = MaterializeSuggestionPayload@@ -92,6 +108,18 @@ class MaterializePayloadSerializer(DataclassSerializer):"help_text": "How old the view's data can be today, in seconds. Null when its sources are live."},"freshness_after_seconds": {"help_text": "How old the data can be once materialized, in seconds."},+ "incremental_key": {+ "help_text": "Date or datetime output column whose advancing value marks new rows. "+ "Null for a full refresh."+ },+ "unique_key": {+ "help_text": "Output columns that identify a row, so a refresh updates rows instead of adding "+ "duplicates. Empty for a full refresh."+ },+ "lookback_seconds": {+ "help_text": "How far before the last refresh's highest incremental_key value each refresh reads "+ "again, in seconds. The last incremental_key value is always read again."+ },}diff --git a/products/warehouse_suggestions/backend/presentation/views.py b/products/warehouse_suggestions/backend/presentation/views.pyindex 942f77b37abc..533d7a0c7e4a 100644--- a/products/warehouse_suggestions/backend/presentation/views.py+++ b/products/warehouse_suggestions/backend/presentation/views.py@@ -37,6 +37,7 @@ from ..facade.contracts import (Suggestion,SuggestionAlreadyDecidedError,SuggestionNotFoundError,+ SuggestionOutdatedError,SuggestionPage,SuggestionSubjectGoneError,)@@ -238,5 +239,7 @@ class WarehouseSuggestionViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSetraise ValidationError({"subject_id": SUBJECT_GONE[error.subject_kind]})except RefreshIntervalRefusedError as error:raise ValidationError({"refresh_interval_seconds": str(error)})+ except SuggestionOutdatedError as error:+ raise Conflict(str(error))except AcceptFailedError as error:raise APIException(str(error))diff --git a/products/warehouse_suggestions/backend/tests/test_accept.py b/products/warehouse_suggestions/backend/tests/test_accept.pyindex ba4045605160..615110e5faa5 100644--- a/products/warehouse_suggestions/backend/tests/test_accept.py+++ b/products/warehouse_suggestions/backend/tests/test_accept.py@@ -22,7 +22,11 @@ from products.warehouse_suggestions.backend.facade.contracts import (SuggestionAlreadyDecidedError,SuggestionPayload,)-from products.warehouse_suggestions.backend.facade.enums import WarehouseSuggestionKind, WarehouseSuggestionStatus+from products.warehouse_suggestions.backend.facade.enums import (+ WarehouseSuggestionKind,+ WarehouseSuggestionRefreshMode,+ WarehouseSuggestionStatus,+)from products.warehouse_suggestions.backend.logic.accept import CertificationAcceptorfrom products.warehouse_suggestions.backend.logic.suggestions import HUMAN_TRANSITIONS, transition_tofrom products.warehouse_suggestions.backend.models import WarehouseSuggestion@@ -33,21 +37,34 @@ from .test_suggestions import make_draftFLAG = "products.warehouse_suggestions.backend.presentation.views.is_warehouse_suggestions_enabled"DAY_SECONDS = 24 * 60 * 60REPORT = "products.warehouse_suggestions.backend.logic.analytics.report_user_or_team_action"+DAILY_COUNTS = "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY day"+DAILY_COUNT_COLUMNS = {+ "day": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')"},+ "c": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt64"},+}+STORED_DAY_KEY = {"enabled": True, "incremental_key": "day", "unique_key": ["day"], "lookback_seconds": 0}+MATERIALIZE_PAYLOAD = MaterializePayload(+ subject_name="orders",+ refresh_interval_seconds=DAY_SECONDS,+ saves_seconds_per_month=900,+ saves_bytes_per_month=0,+ freshness_today_seconds=None,+ freshness_after_seconds=DAY_SECONDS,+ live_sources=(),+ unknown_sources=(),+)+INCREMENTAL_DAY_PAYLOAD = replace(+ MATERIALIZE_PAYLOAD,+ refresh_mode=WarehouseSuggestionRefreshMode.INCREMENTAL,+ incremental_key="day",+ unique_key=("day",),+)PAYLOADS: dict[WarehouseSuggestionKind, SuggestionPayload] = {WarehouseSuggestionKind.CERTIFY: CertifyPayload(subject_name="orders"),WarehouseSuggestionKind.DEPRECATE: DeprecatePayload(subject_name="orders", refresh_seconds_per_month=0, refresh_bytes_per_month=0),- WarehouseSuggestionKind.MATERIALIZE: MaterializePayload(- subject_name="orders",- refresh_interval_seconds=DAY_SECONDS,- saves_seconds_per_month=900,- saves_bytes_per_month=0,- freshness_today_seconds=None,- freshness_after_seconds=DAY_SECONDS,- live_sources=(),- unknown_sources=(),- ),+ WarehouseSuggestionKind.MATERIALIZE: MATERIALIZE_PAYLOAD,}@@ -84,6 +101,10 @@ class TestAcceptSuggestion(APIBaseTest):def _suggest(self, kind: WarehouseSuggestionKind) -> WarehouseSuggestion:return suggest(self.team.id, self.view.id, kind)+ def _suggest_materialize(self, payload: MaterializePayload) -> WarehouseSuggestion:+ draft = make_draft(fingerprint="materialize:orders", subject_id=self.view.id)+ return ingest_surfaced(self.team.id, replace(draft, kind=WarehouseSuggestionKind.MATERIALIZE, payload=payload))+def _accept(self, suggestion_id: UUID, body: dict | None = None) -> dict:response = self.client.post(f"{self.url}/{suggestion_id}/accept/", body or {})return {**response.json(), "http_status": response.status_code}@@ -177,6 +198,50 @@ class TestAcceptSuggestion(APIBaseTest):12 * 60 * 60,)+ @parameterized.expand(+ [+ (+ "query_still_groups_by_the_key",+ DAILY_COUNTS,+ INCREMENTAL_DAY_PAYLOAD,+ status.HTTP_200_OK,+ STORED_DAY_KEY,+ ),+ (+ "query_changed_since_the_suggestion",+ "SELECT timestamp, event FROM events",+ INCREMENTAL_DAY_PAYLOAD,+ status.HTTP_409_CONFLICT,+ STORED_DAY_KEY,+ ),+ (+ "query_no_longer_groups",+ "SELECT toStartOfDay(timestamp) AS day FROM events",+ INCREMENTAL_DAY_PAYLOAD,+ status.HTTP_409_CONFLICT,+ STORED_DAY_KEY,+ ),+ ("full_refresh_clears_leftover_keys", DAILY_COUNTS, MATERIALIZE_PAYLOAD, status.HTTP_200_OK, None),+ ]+ )+ def test_accepting_applies_the_refresh_mode_the_suggestion_shows(+ self,+ _name: str,+ hogql: str,+ payload: MaterializePayload,+ expected_status: int,+ expected_config: dict | None,+ ) -> None:+ DataWarehouseSavedQuery.objects.filter(id=self.view.id).update(+ query={"kind": "HogQLQuery", "query": hogql}, columns=DAILY_COUNT_COLUMNS, incremental_config=STORED_DAY_KEY+ )+ suggestion = self._suggest_materialize(payload)++ accepted = self._accept(suggestion.id)++ self.view.refresh_from_db()+ assert (accepted["http_status"], self.view.incremental_config) == (expected_status, expected_config), accepted+@parameterized.expand([("not_a_schedulable_interval", 7), ("past_the_largest_interval", 10**14)])def test_a_refused_interval_is_a_field_error_and_keeps_the_suggestion_open(self, _name: str, refresh_interval_seconds: intdiff --git a/products/warehouse_suggestions/backend/tests/test_candidates.py b/products/warehouse_suggestions/backend/tests/test_candidates.pyindex 266de9ea0560..e2dc6e625e88 100644--- a/products/warehouse_suggestions/backend/tests/test_candidates.py+++ b/products/warehouse_suggestions/backend/tests/test_candidates.py@@ -3,20 +3,27 @@ from datetime import timedeltafrom uuid import uuid4from posthog.test.base import BaseTest+from unittest.mock import patchfrom django.test import SimpleTestCasefrom parameterized import parameterizedfrom products.data_catalog.backend.facade.enums import CertificationStatus-from products.warehouse_suggestions.backend.facade.contracts import CertifyPayload-from products.warehouse_suggestions.backend.facade.enums import WarehouseSuggestionAssetOutcome+from products.data_modeling.backend.facade.models import DAG, DataWarehouseSavedQuery, Node, NodeType+from products.warehouse_suggestions.backend.facade.contracts import CertifyPayload, MaterializePayload+from products.warehouse_suggestions.backend.facade.enums import (+ WarehouseSuggestionAssetOutcome,+ WarehouseSuggestionRefreshIntervalBasis,+ WarehouseSuggestionRefreshMode,+)from products.warehouse_suggestions.backend.logic.candidates.certify import CertifyCandidatefrom products.warehouse_suggestions.backend.logic.candidates.deprecate import DeprecateCandidatefrom products.warehouse_suggestions.backend.logic.candidates.materialize import (MaterializeCandidate,allowed_intervals,estimate_savings,+ interval_basis,)from products.warehouse_suggestions.backend.logic.rules import RULES, Surfacefrom products.warehouse_suggestions.backend.tests.factories import (@@ -32,6 +39,12 @@ VIEW_ID = uuid4()AT_CERTIFY_FLOORS = {"human_users": 5, "human_days": 20}HOUR = timedelta(hours=1)DAY = timedelta(hours=24)+INCREMENTAL_FLAG = "products.warehouse_suggestions.backend.logic.candidates.materialize.incremental_enabled"+DAILY_COUNTS = "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY day"+DAILY_COUNT_COLUMNS = {+ "day": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')"},+ "c": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt64"},+}class TestCertifyCandidate(SimpleTestCase):@@ -128,12 +141,15 @@ class TestCertifyCandidate(SimpleTestCase):class TestMaterializeRules(SimpleTestCase):@parameterized.expand([- ("read_most_days", 25, [HOUR, timedelta(hours=6), DAY]),- ("read_on_few_days", 10, [DAY]),+ ("read_every_hour_on_most_days", 25, 25 * 24, [HOUR, timedelta(hours=6), DAY]),+ ("read_a_few_times_a_day", 25, 25 * 3, [DAY]),+ ("read_on_few_days", 10, 10 * 24, [DAY]),])- def test_read_cadence_sets_the_shortest_interval(self, _name: str, human_days: int, expected: list) -> None:- reads = busy_reads(human_days=human_days)+ def test_read_cadence_sets_the_shortest_interval(+ self, _name: str, human_days: int, human_requests: int, expected: list+ ) -> None:+ reads = busy_reads(human_days=human_days, human_requests=human_requests)intervals = allowed_intervals(context(team_reads({})), [HOUR, timedelta(hours=6), DAY, timedelta(days=7)], reads@@ -157,18 +173,108 @@ class TestMaterializeRules(SimpleTestCase):@parameterized.expand([- ("limited", "SELECT timestamp, event FROM events ORDER BY timestamp DESC LIMIT 10"),- ("no_key_column", "SELECT count() FROM events"),+ (+ "sources_allow_nothing_finer",+ HOUR,+ [HOUR, DAY],+ True,+ WarehouseSuggestionRefreshIntervalBasis.SOURCE_SYNC,+ ),+ (+ "nothing_refreshes_faster",+ HOUR,+ [HOUR, DAY],+ False,+ WarehouseSuggestionRefreshIntervalBasis.FASTEST_SCHEDULE,+ ),+ ("reads_allow_nothing_finer", DAY, [DAY], False, WarehouseSuggestionRefreshIntervalBasis.READ_RATE),+ (+ "finer_ones_save_too_little",+ DAY,+ [timedelta(hours=6), DAY],+ False,+ WarehouseSuggestionRefreshIntervalBasis.SAVINGS,+ ),])- def test_a_view_that_cannot_refresh_incrementally_is_not_proposed(self, _name: str, hogql: str) -> None:- reads = team_reads({view_subject(VIEW_ID): busy_reads()})- not_incremental = view(VIEW_ID, hogql=hogql)+ def test_interval_basis_names_what_kept_the_interval_from_being_shorter(+ self,+ _name: str,+ interval: timedelta,+ read_allowed: list[timedelta],+ limited_by_sources: bool,+ expected: WarehouseSuggestionRefreshIntervalBasis,+ ) -> None:+ basis = interval_basis(+ interval, [HOUR, timedelta(hours=6), DAY], read_allowed, limited_by_sources=limited_by_sources+ )++ assert basis == expected- result = MaterializeCandidate().evaluate(context(reads, views=[not_incremental]))- assert result.drafts == ()- assert [rejection.reason for rejection in result.rejections] == ["its query cannot refresh incrementally"]+class TestMaterializeRefreshMode(BaseTest):+ @parameterized.expand(+ [+ ("grouped_by_day", DAILY_COUNTS, True, WarehouseSuggestionRefreshMode.INCREMENTAL, "day", ("day",)),+ ("flag_off", DAILY_COUNTS, False, WarehouseSuggestionRefreshMode.FULL, None, ()),+ (+ "limited_query",+ "SELECT timestamp, event FROM events ORDER BY timestamp DESC LIMIT 10",+ True,+ WarehouseSuggestionRefreshMode.FULL,+ None,+ (),+ ),+ (+ "grouped_over_a_nondeterministic_view",+ DAILY_COUNTS.replace("FROM events", "FROM sampled_events"),+ True,+ WarehouseSuggestionRefreshMode.FULL,+ None,+ (),+ ),+ ]+ )+ def test_a_suggestion_proposes_incremental_only_when_the_keys_are_certain(+ self,+ _name: str,+ hogql: str,+ flag_on: bool,+ refresh_mode: WarehouseSuggestionRefreshMode,+ incremental_key: str | None,+ unique_key: tuple[str, ...],+ ) -> None:+ DataWarehouseSavedQuery.objects.create(+ team=self.team,+ name="sampled_events",+ query={"kind": "HogQLQuery", "query": "SELECT timestamp FROM events WHERE rand() % 10 = 0"},+ columns={"timestamp": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')"}},+ )+ saved_query = DataWarehouseSavedQuery.objects.create(+ team=self.team,+ name="daily_counts",+ query={"kind": "HogQLQuery", "query": hogql},+ columns=DAILY_COUNT_COLUMNS,+ )+ dag = DAG.objects.create(team=self.team, name="Default")+ Node.objects.create(team=self.team, dag=dag, name="daily_counts", saved_query=saved_query, type=NodeType.VIEW)+ candidate_context = context(+ team_reads({view_subject(saved_query.id): busy_reads(human_duration_ms=200 * 10_000)}),+ views=[view(saved_query.id, name="daily_counts", hogql=hogql)],+ team_id=self.team.pk,+ )++ with patch(INCREMENTAL_FLAG, return_value=flag_on):+ [draft] = MaterializeCandidate().evaluate(candidate_context).drafts++ assert isinstance(draft.payload, MaterializePayload)+ expected_lookback = RULES.materialize.incremental_lookback_seconds if incremental_key else 0+ assert (+ draft.payload.refresh_mode,+ draft.payload.incremental_key,+ draft.payload.unique_key,+ draft.payload.lookback_seconds,+ ) == (refresh_mode, incremental_key, unique_key, expected_lookback)class TestDeprecateCandidate(SimpleTestCase):diff --git a/products/warehouse_suggestions/backend/tests/test_job.py b/products/warehouse_suggestions/backend/tests/test_job.pyindex 60078f8ebf89..c32bfcd38e54 100644--- a/products/warehouse_suggestions/backend/tests/test_job.py+++ b/products/warehouse_suggestions/backend/tests/test_job.py@@ -2,6 +2,7 @@ from datetime import UTC, date, datetime, time, timedeltafrom uuid import uuid4from posthog.test.base import BaseTest, ClickhouseTestMixin+from unittest.mock import patchfrom parameterized import parameterized@@ -22,6 +23,12 @@ TODAY = date.today()NOW = datetime.combine(TODAY, time(hour=9), tzinfo=UTC)BEFORE_THE_WINDOW = NOW - timedelta(days=2 * RULES.window_days)SCENES = ("SQLEditor", "Dashboard")+INCREMENTAL_FLAG = "products.warehouse_suggestions.backend.logic.candidates.materialize.incremental_enabled"+DAILY_COUNTS = "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY day"+DAILY_COUNT_COLUMNS = {+ "day": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')"},+ "c": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt64"},+}FULL_ROLLUP = RollupDays(days_with_data=RULES.window_days, recent_days_with_data=RULES.lifecycle.expire_after_days)@@ -47,6 +54,9 @@ class TestRunTeam(ClickhouseTestMixin, BaseTest):def test_proposes_each_kind_from_a_month_of_reads_and_surfaces_them(self) -> None:dag = DAG.objects.create(team=self.team, name="Default")busy, busy_node = self._view("busy_orders", dag)+ DataWarehouseSavedQuery.objects.filter(id=busy.id).update(+ query={"kind": "HogQLQuery", "query": DAILY_COUNTS}, columns=DAILY_COUNT_COLUMNS+ )for name, properties in [("events", {"origin": "posthog"}), ("legacy_orders", {})]:table = Node.objects.create(team=self.team, dag=dag, name=name, type=NodeType.TABLE, properties=properties)Edge.objects.create(team=self.team, dag=dag, source=table, target=busy_node)@@ -72,7 +82,8 @@ class TestRunTeam(ClickhouseTestMixin, BaseTest):],)- result = run_team(self.team.pk, run_id="run-1", today=TODAY, rollup_days=FULL_ROLLUP, now=NOW)+ with patch(INCREMENTAL_FLAG, return_value=True):+ result = run_team(self.team.pk, run_id="run-1", today=TODAY, rollup_days=FULL_ROLLUP, now=NOW)assert result.status == TeamRunStatus.PROCESSEDassert sorted(@@ -93,6 +104,11 @@ class TestRunTeam(ClickhouseTestMixin, BaseTest):materialize = WarehouseSuggestion.objects.for_team(self.team.pk).get(kind=WarehouseSuggestionKind.MATERIALIZE)assert (materialize.payload["refresh_interval_seconds"], materialize.run_id) == (24 * 60 * 60, "run-1")assert [source["name"] for source in materialize.payload["live_sources"]] == ["events"]+ assert (+ materialize.payload["refresh_mode"],+ materialize.payload["incremental_key"],+ materialize.payload["unique_key"],+ ) == ("incremental", "day", ["day"])@parameterized.expand([diff --git a/products/warehouse_suggestions/frontend/generated/api.schemas.ts b/products/warehouse_suggestions/frontend/generated/api.schemas.tsindex 7aadd203da18..0037b6714563 100644--- a/products/warehouse_suggestions/frontend/generated/api.schemas.ts+++ b/products/warehouse_suggestions/frontend/generated/api.schemas.ts@@ -28,11 +28,51 @@ export interface WarehouseSuggestionVisibleSourcesApi {hidden_count: number}+/**+ * * `full` - Full+ * * `incremental` - Incremental+ */+export type WarehouseSuggestionRefreshModeEnumApi =+ (typeof WarehouseSuggestionRefreshModeEnumApi)[keyof typeof WarehouseSuggestionRefreshModeEnumApi]++export const WarehouseSuggestionRefreshModeEnumApi = {+ Full: 'full',+ Incremental: 'incremental',+} as const++/**+ * * `source_sync` - Source sync+ * * `fastest_schedule` - Fastest schedule+ * * `read_rate` - Read rate+ * * `savings` - Savings+ */+export type WarehouseSuggestionRefreshIntervalBasisEnumApi =+ (typeof WarehouseSuggestionRefreshIntervalBasisEnumApi)[keyof typeof WarehouseSuggestionRefreshIntervalBasisEnumApi]++export const WarehouseSuggestionRefreshIntervalBasisEnumApi = {+ SourceSync: 'source_sync',+ FastestSchedule: 'fastest_schedule',+ ReadRate: 'read_rate',+ Savings: 'savings',+} as const+export interface WarehouseSuggestionMaterializePayloadApi {/** Sources that are always current, such as PostHog tables and direct connections. */live_sources: WarehouseSuggestionVisibleSourcesApi/** Sources with no sync schedule, so their freshness is unknown. */unknown_sources: WarehouseSuggestionVisibleSourcesApi+ /** incremental: each refresh reads only rows with a new incremental_key value and updates rows by unique_key. full: each refresh rebuilds the whole table. Incremental is proposed only when the keys are certain from the query.+ *+ * * `full` - Full+ * * `incremental` - Incremental */+ refresh_mode: WarehouseSuggestionRefreshModeEnumApi+ /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.+ *+ * * `source_sync` - Source sync+ * * `fastest_schedule` - Fastest schedule+ * * `read_rate` - Read rate+ * * `savings` - Savings */+ refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasisEnumApi | null/** Name of the view to materialize. */subject_name: string/** Proposed refresh interval, in seconds. */@@ -48,6 +88,15 @@ export interface WarehouseSuggestionMaterializePayloadApi {freshness_today_seconds: number | null/** How old the data can be once materialized, in seconds. */freshness_after_seconds: number+ /**+ * Date or datetime output column whose advancing value marks new rows. Null for a full refresh.+ * @nullable+ */+ incremental_key?: string | null+ /** Output columns that identify a row, so a refresh updates rows instead of adding duplicates. Empty for a full refresh. */+ unique_key?: string[]+ /** How far before the last refresh's highest incremental_key value each refresh reads again, in seconds. The last incremental_key value is always read again. */+ lookback_seconds?: number}export type WarehouseSuggestionPayloadApi =diff --git a/products/warehouse_suggestions/mcp/tools.yaml b/products/warehouse_suggestions/mcp/tools.yamlindex 9218ef366f35..c90ec63771ed 100644--- a/products/warehouse_suggestions/mcp/tools.yaml+++ b/products/warehouse_suggestions/mcp/tools.yaml@@ -24,7 +24,9 @@ tools:Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table ascertified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view onthe proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another- schedule. Requires human typed confirmation.+ schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by+ incremental_key and update rows by unique_key, full refreshes rebuild the whole table. A 409 means the view+ changed since the suggestion was made and its keys no longer apply. Requires human typed confirmation.confirmed_action:message: >About to accept warehouse suggestion '{id}'. This applies the change it proposes to the view or table.diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex 136d53f8d150..d79562411ea2 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -14752,7 +14752,7 @@}},"warehouse-suggestions-accept-execute": {- "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. Requires human typed confirmation.",+ "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. A 409 means the view changed since the suggestion was made and its keys no longer apply. Requires human typed confirmation.","category": "Warehouse suggestions","feature": "warehouse_suggestions","summary": "Accept a warehouse suggestion (execute)",@@ -14767,7 +14767,7 @@"feature_flag": "warehouse-suggestions"},"warehouse-suggestions-accept-prepare": {- "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. Requires human typed confirmation.",+ "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. A 409 means the view changed since the suggestion was made and its keys no longer apply. Requires human typed confirmation.","category": "Warehouse suggestions","feature": "warehouse_suggestions","summary": "Accept a warehouse suggestion (prepare)",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex a20cce51ece9..3a345f1518a6 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -15446,7 +15446,7 @@}},"warehouse-suggestions-accept-execute": {- "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. Requires human typed confirmation.",+ "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. A 409 means the view changed since the suggestion was made and its keys no longer apply. Requires human typed confirmation.","category": "Warehouse suggestions","feature": "warehouse_suggestions","summary": "Accept a warehouse suggestion (execute)",@@ -15461,7 +15461,7 @@"feature_flag": "warehouse-suggestions"},"warehouse-suggestions-accept-prepare": {- "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. Requires human typed confirmation.",+ "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. A 409 means the view changed since the suggestion was made and its keys no longer apply. Requires human typed confirmation.","category": "Warehouse suggestions","feature": "warehouse_suggestions","summary": "Accept a warehouse suggestion (prepare)",diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 2063167f66f7..58933b75857e 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -78918,11 +78918,51 @@ export namespace Schemas {hidden_count: number;}+ /**+ * * `full` - Full+ * * `incremental` - Incremental+ */+ export type WarehouseSuggestionRefreshModeEnum = typeof WarehouseSuggestionRefreshModeEnum[keyof typeof WarehouseSuggestionRefreshModeEnum];+++ export const WarehouseSuggestionRefreshModeEnum = {+ Full: 'full',+ Incremental: 'incremental',+ } as const;++ /**+ * * `source_sync` - Source sync+ * * `fastest_schedule` - Fastest schedule+ * * `read_rate` - Read rate+ * * `savings` - Savings+ */+ export type WarehouseSuggestionRefreshIntervalBasisEnum = typeof WarehouseSuggestionRefreshIntervalBasisEnum[keyof typeof WarehouseSuggestionRefreshIntervalBasisEnum];+++ export const WarehouseSuggestionRefreshIntervalBasisEnum = {+ SourceSync: 'source_sync',+ FastestSchedule: 'fastest_schedule',+ ReadRate: 'read_rate',+ Savings: 'savings',+ } as const;+export interface WarehouseSuggestionMaterializePayload {/** Sources that are always current, such as PostHog tables and direct connections. */live_sources: WarehouseSuggestionVisibleSources;/** Sources with no sync schedule, so their freshness is unknown. */unknown_sources: WarehouseSuggestionVisibleSources;+ /** incremental: each refresh reads only rows with a new incremental_key value and updates rows by unique_key. full: each refresh rebuilds the whole table. Incremental is proposed only when the keys are certain from the query.+ *+ * * `full` - Full+ * * `incremental` - Incremental */+ refresh_mode: WarehouseSuggestionRefreshModeEnum;+ /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.+ *+ * * `source_sync` - Source sync+ * * `fastest_schedule` - Fastest schedule+ * * `read_rate` - Read rate+ * * `savings` - Savings */+ refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasisEnum | null;/** Name of the view to materialize. */subject_name: string;/** Proposed refresh interval, in seconds. */@@ -78938,6 +78978,15 @@ export namespace Schemas {freshness_today_seconds: number | null;/** How old the data can be once materialized, in seconds. */freshness_after_seconds: number;+ /**+ * Date or datetime output column whose advancing value marks new rows. Null for a full refresh.+ * @nullable+ */+ incremental_key?: string | null;+ /** Output columns that identify a row, so a refresh updates rows instead of adding duplicates. Empty for a full refresh. */+ unique_key?: string[];+ /** How far before the last refresh's highest incremental_key value each refresh reads again, in seconds. The last incremental_key value is always read again. */+ lookback_seconds?: number;}export type WarehouseSuggestionPayload = WarehouseSuggestionCertifyPayload | WarehouseSuggestionDeprecatePayload | WarehouseSuggestionMaterializePayload;