Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-suggestions): propose the refresh mode and freshness (#119703)

Thiago Salvatore593fa0f6bb3e
diff --git a/products/warehouse_suggestions/backend/facade/contracts.py b/products/warehouse_suggestions/backend/facade/contracts.py
index a43224782d4b..158e987d22af 100644
--- a/products/warehouse_suggestions/backend/facade/contracts.py
+++ b/products/warehouse_suggestions/backend/facade/contracts.py
@@ -10,6 +10,8 @@ from .enums import (
WarehouseSuggestionAssetOutcome,
WarehouseSuggestionDismissalReason,
WarehouseSuggestionKind,
+ WarehouseSuggestionRefreshIntervalBasis,
+ WarehouseSuggestionRefreshMode,
WarehouseSuggestionStatus,
WarehouseSuggestionSubjectKind,
)
@@ -53,6 +55,11 @@ class MaterializePayload:
freshness_after_seconds: int
live_sources: tuple[SourceRef, ...]
unknown_sources: tuple[SourceRef, ...]
+ refresh_mode: WarehouseSuggestionRefreshMode = WarehouseSuggestionRefreshMode.FULL
+ incremental_key: str | None = None
+ unique_key: tuple[str, ...] = ()
+ lookback_seconds: int = 0
+ refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasis | None = None
SuggestionPayload = CertifyPayload | DeprecatePayload | MaterializePayload
@@ -76,6 +83,11 @@ class MaterializeSuggestionPayload:
freshness_after_seconds: int
live_sources: VisibleSources
unknown_sources: VisibleSources
+ refresh_mode: WarehouseSuggestionRefreshMode = WarehouseSuggestionRefreshMode.FULL
+ incremental_key: str | None = None
+ unique_key: tuple[str, ...] = ()
+ lookback_seconds: int = 0
+ refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasis | None = None
SuggestionPayloadView = CertifyPayload | DeprecatePayload | MaterializeSuggestionPayload
@@ -198,5 +210,9 @@ class RefreshIntervalRefusedError(Exception):
pass
+class SuggestionOutdatedError(Exception):
+ pass
+
+
class AcceptFailedError(Exception):
pass
diff --git a/products/warehouse_suggestions/backend/facade/enums.py b/products/warehouse_suggestions/backend/facade/enums.py
index 3c1529d2b104..c449d9d0ba05 100644
--- a/products/warehouse_suggestions/backend/facade/enums.py
+++ b/products/warehouse_suggestions/backend/facade/enums.py
@@ -32,3 +32,15 @@ class WarehouseSuggestionAssetOutcome(LabeledStrEnum):
LIVE = "live"
DELETED = "deleted"
UNUSED = "unused"
+
+
+class WarehouseSuggestionRefreshMode(LabeledStrEnum):
+ FULL = "full"
+ INCREMENTAL = "incremental"
+
+
+class WarehouseSuggestionRefreshIntervalBasis(LabeledStrEnum):
+ SOURCE_SYNC = ("source_sync", "Source sync")
+ FASTEST_SCHEDULE = ("fastest_schedule", "Fastest schedule")
+ READ_RATE = ("read_rate", "Read rate")
+ SAVINGS = "savings"
diff --git a/products/warehouse_suggestions/backend/logic/accept.py b/products/warehouse_suggestions/backend/logic/accept.py
index adb971dedca2..39ab2b1aeb27 100644
--- a/products/warehouse_suggestions/backend/logic/accept.py
+++ b/products/warehouse_suggestions/backend/logic/accept.py
@@ -10,12 +10,18 @@ from posthog.dataclasses import frozen
from products.data_catalog.backend.facade.api import certifications_for_team, certify, deprecate, propose_certification
from products.data_catalog.backend.facade.enums import CertificationStatus
from products.data_modeling.backend.facade.api import (
+ IncrementalConfig,
+ IncrementalConfigRefusedError,
MaterializationFailedError,
MaterializationForbiddenError,
MaterializationRefusedError,
SavedQueryNotFoundError,
+ definition_database,
enable_saved_query_materialization,
+ get_saved_query_columns,
+ get_saved_query_sql,
get_saved_query_summary,
+ propose_incremental_config,
)
from products.warehouse_sources.backend.facade.api import get_queryable_table
@@ -29,10 +35,16 @@ from ..facade.contracts import (
SubjectAlreadyCertifiedError,
SubjectEditAccessRequiredError,
SuggestionAlreadyDecidedError,
+ SuggestionOutdatedError,
SuggestionPayload,
SuggestionSubjectGoneError,
)
-from ..facade.enums import WarehouseSuggestionKind, WarehouseSuggestionStatus, WarehouseSuggestionSubjectKind
+from ..facade.enums import (
+ WarehouseSuggestionKind,
+ WarehouseSuggestionRefreshMode,
+ WarehouseSuggestionStatus,
+ WarehouseSuggestionSubjectKind,
+)
from ..models import WarehouseSuggestion
from .analytics import SuggestionOutcome, report_outcomes
from .payloads import payload_from_json
@@ -94,6 +106,11 @@ class CertificationAcceptor(Acceptor):
return existing
+VIEW_CHANGED_SINCE_SUGGESTED = (
+ "This view changed after the suggestion was made, so it can no longer refresh incrementally on the suggested keys."
+)
+
+
class MaterializeAcceptor(Acceptor):
def accept(
self, suggestion: WarehouseSuggestion, payload: SuggestionPayload, request: AcceptRequest
@@ -101,6 +118,9 @@ class MaterializeAcceptor(Acceptor):
if not isinstance(payload, MaterializePayload):
raise AcceptFailedError("This suggestion has no refresh interval to apply.")
interval = request.refresh_interval or timedelta(seconds=payload.refresh_interval_seconds)
+ incremental_config = _incremental_config(payload)
+ if incremental_config is not None and not _still_proposed(suggestion, incremental_config, request.user):
+ raise SuggestionOutdatedError(VIEW_CHANGED_SINCE_SUGGESTED)
try:
enable_saved_query_materialization(
suggestion.team_id,
@@ -109,11 +129,14 @@ class MaterializeAcceptor(Acceptor):
sync_frequency_interval=interval,
visible_blocker_names=_no_blocker_names,
was_impersonated=request.was_impersonated,
+ incremental_config=incremental_config,
)
except SavedQueryNotFoundError:
raise SuggestionSubjectGoneError(WarehouseSuggestionSubjectKind.SAVED_QUERY)
except MaterializationForbiddenError:
raise SubjectEditAccessRequiredError(WarehouseSuggestionSubjectKind.SAVED_QUERY)
+ except IncrementalConfigRefusedError as error:
+ raise SuggestionOutdatedError(f"{VIEW_CHANGED_SINCE_SUGGESTED} {error}")
except MaterializationRefusedError as error:
raise RefreshIntervalRefusedError(str(error))
except MaterializationFailedError as error:
@@ -123,6 +146,29 @@ class MaterializeAcceptor(Acceptor):
)
+def _incremental_config(payload: MaterializePayload) -> IncrementalConfig | None:
+ if payload.refresh_mode != WarehouseSuggestionRefreshMode.INCREMENTAL or payload.incremental_key is None:
+ return None
+ return IncrementalConfig(
+ incremental_key=payload.incremental_key,
+ unique_key=payload.unique_key,
+ lookback_seconds=payload.lookback_seconds,
+ )
+
+
+def _still_proposed(suggestion: WarehouseSuggestion, config: IncrementalConfig, user: "User") -> bool:
+ proposed = propose_incremental_config(
+ get_saved_query_sql(suggestion.team_id, suggestion.subject_id) or "",
+ get_saved_query_columns(suggestion.team_id, suggestion.subject_id),
+ database=definition_database(suggestion.team_id, user=user),
+ )
+ return (
+ proposed is not None
+ and proposed.incremental_key == config.incremental_key
+ and set(proposed.unique_key) == set(config.unique_key)
+ )
+
+
ACCEPTORS: Mapping[WarehouseSuggestionKind, Acceptor] = {
WarehouseSuggestionKind.CERTIFY: CertificationAcceptor(
status=CertificationStatus.CERTIFIED, apply=certify, reuses_existing=False
diff --git a/products/warehouse_suggestions/backend/logic/candidates/materialize.py b/products/warehouse_suggestions/backend/logic/candidates/materialize.py
index 8fa07332736b..a409a9ef5190 100644
--- a/products/warehouse_suggestions/backend/logic/candidates/materialize.py
+++ b/products/warehouse_suggestions/backend/logic/candidates/materialize.py
@@ -1,21 +1,35 @@
from collections.abc import Iterable
+from dataclasses import replace
from datetime import datetime, timedelta
+from functools import cached_property
from uuid import UUID
+from posthog.hogql.database.database import Database
+
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from products.data_modeling.backend.facade.api import (
+ IncrementalConfig,
MissingDagNodeError,
SavedQueryFrequencyBounds,
- check_incremental_eligibility,
+ definition_database,
+ get_saved_query_columns,
+ incremental_enabled,
+ propose_incremental_config,
saved_query_target_bounds,
upstream_table_refs,
)
from products.data_modeling.backend.facade.contracts import SavedQueryDefinition
from ...facade.contracts import MaterializePayload, SourceRef
-from ...facade.enums import WarehouseSuggestionAssetOutcome, WarehouseSuggestionKind, WarehouseSuggestionSubjectKind
+from ...facade.enums import (
+ WarehouseSuggestionAssetOutcome,
+ WarehouseSuggestionKind,
+ WarehouseSuggestionRefreshIntervalBasis,
+ WarehouseSuggestionRefreshMode,
+ WarehouseSuggestionSubjectKind,
+)
from ..reads import Subject, SubjectReads
from .base import DAYS_PER_MONTH, MILLISECONDS_PER_SECOND, Candidate, CandidateContext, CandidateResult, Rejection
@@ -43,18 +57,44 @@ class _Proposal:
estimate: SavingsEstimate
+class _IncrementalProposer:
+ """Proposes incremental keys for one team run, reading the flag and building the team database at most once."""
+
+ def __init__(self, team_id: int) -> None:
+ self._team_id = team_id
+
+ @cached_property
+ def _enabled(self) -> bool:
+ return incremental_enabled(self._team_id)
+
+ @cached_property
+ def _database(self) -> Database | None:
+ try:
+ return definition_database(self._team_id)
+ except Exception as error:
+ capture_exception(error)
+ return None
+
+ def config_for(self, saved_query: SavedQueryDefinition) -> IncrementalConfig | None:
+ if not self._enabled or self._database is None:
+ return None
+ column_types = get_saved_query_columns(self._team_id, saved_query.id)
+ return propose_incremental_config(saved_query.hogql, column_types, database=self._database)
+
+
class MaterializeCandidate(Candidate):
kind = WarehouseSuggestionKind.MATERIALIZE
def evaluate(self, context: CandidateContext) -> CandidateResult:
drafts = []
rejections = []
+ incremental = _IncrementalProposer(context.team_id)
for saved_query in context.inventory.saved_queries.values():
subject = Subject(kind=WarehouseSuggestionSubjectKind.SAVED_QUERY, id=saved_query.id)
reads = context.reads.reads_of(subject)
if reads is None or reads.human_requests == 0 or not self._is_unmaterialized(context, saved_query):
continue
- outcome = self._propose(context, saved_query, reads)
+ outcome = self._propose(context, saved_query, reads, incremental)
if isinstance(outcome, str):
rejections.append(Rejection(subject=subject, reason=outcome))
continue
@@ -92,15 +132,15 @@ class MaterializeCandidate(Candidate):
return not saved_query.materializes and context.is_suggestible_view(saved_query)
def _propose(
- self, context: CandidateContext, saved_query: SavedQueryDefinition, reads: SubjectReads
+ self,
+ context: CandidateContext,
+ saved_query: SavedQueryDefinition,
+ reads: SubjectReads,
+ incremental: _IncrementalProposer,
) -> _Proposal | str:
rejection = self._read_floor_rejection(context, reads)
if rejection is not None:
return rejection
- if context.rules.materialize.require_incremental_eligibility and not _can_refresh_incrementally(
- saved_query.hogql
- ):
- return "its query cannot refresh incrementally"
frequency = saved_query_target_bounds(context.team_id, saved_query.id)
if frequency is None:
capture_exception(MissingDagNodeError(f"Saved query {saved_query.id} has no DAG node"))
@@ -121,16 +161,25 @@ class MaterializeCandidate(Candidate):
sources = _classify_sources(context, saved_query.id, frequency)
freshness_today = int(frequency.max_data_age.total_seconds()) or None
interval_seconds = int(estimate.interval.total_seconds())
+ payload = MaterializePayload(
+ subject_name=saved_query.name,
+ refresh_interval_seconds=interval_seconds,
+ saves_seconds_per_month=estimate.seconds_saved,
+ saves_bytes_per_month=estimate.bytes_saved,
+ freshness_today_seconds=freshness_today,
+ freshness_after_seconds=interval_seconds + (freshness_today or 0),
+ live_sources=sources.live,
+ unknown_sources=sources.unknown,
+ refresh_interval_basis=interval_basis(
+ estimate.interval,
+ frequency.bounds.allowed,
+ intervals,
+ limited_by_sources=frequency.bounds.floor is not None,
+ ),
+ )
return _Proposal(
- payload=MaterializePayload(
- subject_name=saved_query.name,
- refresh_interval_seconds=interval_seconds,
- saves_seconds_per_month=estimate.seconds_saved,
- saves_bytes_per_month=estimate.bytes_saved,
- freshness_today_seconds=freshness_today,
- freshness_after_seconds=interval_seconds + (freshness_today or 0),
- live_sources=sources.live,
- unknown_sources=sources.unknown,
+ payload=_with_refresh_mode(
+ payload, incremental.config_for(saved_query), context.rules.materialize.incremental_lookback_seconds
),
estimate=estimate,
)
@@ -150,18 +199,43 @@ class MaterializeCandidate(Candidate):
return None
-def _can_refresh_incrementally(hogql: str) -> bool:
- eligibility = check_incremental_eligibility(hogql, None)
- return eligibility.eligible and bool(eligibility.key_candidates) and bool(eligibility.unique_key_candidates)
-
-
def allowed_intervals(context: CandidateContext, allowed: Iterable[timedelta], reads: SubjectReads) -> list[timedelta]:
refresh = context.rules.refresh
intervals = sorted(interval for interval in allowed if interval <= refresh.max_interval)
read_day_share = reads.human_days / context.days_observed
if read_day_share < refresh.min_read_day_share_below_max_interval:
return [interval for interval in intervals if interval >= refresh.max_interval]
- return intervals
+ return [interval for interval in intervals if interval >= read_interval(reads)]
+
+
+def read_interval(reads: SubjectReads) -> timedelta:
+ return timedelta(days=reads.human_days) / max(reads.human_requests, 1)
+
+
+def interval_basis(
+ interval: timedelta, allowed: Iterable[timedelta], read_allowed: list[timedelta], *, limited_by_sources: bool
+) -> WarehouseSuggestionRefreshIntervalBasis:
+ if interval == min(allowed):
+ if limited_by_sources:
+ return WarehouseSuggestionRefreshIntervalBasis.SOURCE_SYNC
+ return WarehouseSuggestionRefreshIntervalBasis.FASTEST_SCHEDULE
+ if interval == read_allowed[0]:
+ return WarehouseSuggestionRefreshIntervalBasis.READ_RATE
+ return WarehouseSuggestionRefreshIntervalBasis.SAVINGS
+
+
+def _with_refresh_mode(
+ payload: MaterializePayload, config: IncrementalConfig | None, lookback_seconds: int
+) -> MaterializePayload:
+ if config is None:
+ return payload
+ return replace(
+ payload,
+ refresh_mode=WarehouseSuggestionRefreshMode.INCREMENTAL,
+ incremental_key=config.incremental_key,
+ unique_key=config.unique_key,
+ lookback_seconds=lookback_seconds,
+ )
def estimate_savings(context: CandidateContext, reads: SubjectReads, interval: timedelta) -> SavingsEstimate:
diff --git a/products/warehouse_suggestions/backend/logic/payloads.py b/products/warehouse_suggestions/backend/logic/payloads.py
index a759816541ea..fa2f8a617c29 100644
--- a/products/warehouse_suggestions/backend/logic/payloads.py
+++ b/products/warehouse_suggestions/backend/logic/payloads.py
@@ -1,6 +1,6 @@
import json
from collections.abc import Collection, Iterable, Mapping
-from dataclasses import asdict
+from dataclasses import asdict, fields
from typing import Any
from uuid import UUID
@@ -16,7 +16,11 @@ from ..facade.contracts import (
UnsupportedPayloadVersionError,
VisibleSources,
)
-from ..facade.enums import WarehouseSuggestionKind
+from ..facade.enums import (
+ WarehouseSuggestionKind,
+ WarehouseSuggestionRefreshIntervalBasis,
+ WarehouseSuggestionRefreshMode,
+)
SOURCE_FIELDS = ("live_sources", "unknown_sources")
@@ -35,7 +39,7 @@ def payload_from_json(
if kind == WarehouseSuggestionKind.DEPRECATE:
return DeprecatePayload(**data)
sources = {field: tuple(_source_ref(source) for source in data[field]) for field in SOURCE_FIELDS}
- return MaterializePayload(**{**data, **sources})
+ return MaterializePayload(**{**data, **sources, **_refresh_fields(data)})
def source_table_ids(payloads: Iterable[SuggestionPayload]) -> set[UUID]:
@@ -51,13 +55,9 @@ def source_table_ids(payloads: Iterable[SuggestionPayload]) -> set[UUID]:
def payload_view(payload: SuggestionPayload, readable_table_ids: Collection[UUID]) -> SuggestionPayloadView:
if not isinstance(payload, MaterializePayload):
return payload
+ shared = {field.name: getattr(payload, field.name) for field in fields(payload) if field.name not in SOURCE_FIELDS}
return MaterializeSuggestionPayload(
- subject_name=payload.subject_name,
- refresh_interval_seconds=payload.refresh_interval_seconds,
- saves_seconds_per_month=payload.saves_seconds_per_month,
- saves_bytes_per_month=payload.saves_bytes_per_month,
- freshness_today_seconds=payload.freshness_today_seconds,
- freshness_after_seconds=payload.freshness_after_seconds,
+ **shared,
live_sources=_visible_sources(payload.live_sources, readable_table_ids),
unknown_sources=_visible_sources(payload.unknown_sources, readable_table_ids),
)
@@ -75,3 +75,12 @@ def _visible_sources(sources: tuple[SourceRef, ...], readable_table_ids: Collect
def _source_ref(source: Mapping[str, Any]) -> SourceRef:
table_id = source["warehouse_table_id"]
return SourceRef(name=source["name"], warehouse_table_id=UUID(table_id) if table_id else None)
+
+
+def _refresh_fields(data: Mapping[str, Any]) -> dict[str, Any]:
+ basis = data.get("refresh_interval_basis")
+ return {
+ "refresh_mode": WarehouseSuggestionRefreshMode(data.get("refresh_mode", WarehouseSuggestionRefreshMode.FULL)),
+ "unique_key": tuple(data.get("unique_key", ())),
+ "refresh_interval_basis": WarehouseSuggestionRefreshIntervalBasis(basis) if basis else None,
+ }
diff --git a/products/warehouse_suggestions/backend/logic/rules.py b/products/warehouse_suggestions/backend/logic/rules.py
index c6c38c978ab0..9a110781ec9c 100644
--- a/products/warehouse_suggestions/backend/logic/rules.py
+++ b/products/warehouse_suggestions/backend/logic/rules.py
@@ -102,7 +102,7 @@ class MaterializeRules:
min_seconds_saved: float
min_bytes_saved: float
min_saved_share: float
- require_incremental_eligibility: bool
+ incremental_lookback_seconds: int
@frozen
@@ -201,7 +201,7 @@ RULES = Rules(
min_seconds_saved=600.0,
min_bytes_saved=float(TERABYTE),
min_saved_share=0.5,
- require_incremental_eligibility=True,
+ incremental_lookback_seconds=int(timedelta(days=3).total_seconds()),
),
refresh=RefreshRules(max_interval=timedelta(hours=24), min_read_day_share_below_max_interval=0.5),
lifecycle=LifecycleRules(
diff --git a/products/warehouse_suggestions/backend/presentation/serializers.py b/products/warehouse_suggestions/backend/presentation/serializers.py
index 44e0c41679d9..95ed4f34c2e6 100644
--- a/products/warehouse_suggestions/backend/presentation/serializers.py
+++ b/products/warehouse_suggestions/backend/presentation/serializers.py
@@ -21,6 +21,8 @@ from ..facade.enums import (
WarehouseSuggestionAssetOutcome,
WarehouseSuggestionDismissalReason,
WarehouseSuggestionKind,
+ WarehouseSuggestionRefreshIntervalBasis,
+ WarehouseSuggestionRefreshMode,
WarehouseSuggestionStatus,
WarehouseSuggestionSubjectKind,
)
@@ -80,6 +82,20 @@ class MaterializePayloadSerializer(DataclassSerializer):
unknown_sources = VisibleSourcesSerializer(
help_text="Sources with no sync schedule, so their freshness is unknown."
)
+ refresh_mode = serializers.ChoiceField(
+ choices=WarehouseSuggestionRefreshMode.choices,
+ help_text="incremental: each refresh reads only rows with a new incremental_key value and updates rows by "
+ "unique_key. full: each refresh rebuilds the whole table. Incremental is proposed only when the keys are "
+ "certain from the query.",
+ )
+ refresh_interval_basis = serializers.ChoiceField(
+ choices=WarehouseSuggestionRefreshIntervalBasis.choices,
+ allow_null=True,
+ help_text="Why the interval is not shorter. source_sync: the view's sources sync no faster. "
+ "fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. "
+ "savings: a shorter interval would cost more than it saves. Null on suggestions made before this was "
+ "recorded.",
+ )
class Meta:
dataclass = MaterializeSuggestionPayload
@@ -92,6 +108,18 @@ class MaterializePayloadSerializer(DataclassSerializer):
"help_text": "How old the view's data can be today, in seconds. Null when its sources are live."
},
"freshness_after_seconds": {"help_text": "How old the data can be once materialized, in seconds."},
+ "incremental_key": {
+ "help_text": "Date or datetime output column whose advancing value marks new rows. "
+ "Null for a full refresh."
+ },
+ "unique_key": {
+ "help_text": "Output columns that identify a row, so a refresh updates rows instead of adding "
+ "duplicates. Empty for a full refresh."
+ },
+ "lookback_seconds": {
+ "help_text": "How far before the last refresh's highest incremental_key value each refresh reads "
+ "again, in seconds. The last incremental_key value is always read again."
+ },
}
diff --git a/products/warehouse_suggestions/backend/presentation/views.py b/products/warehouse_suggestions/backend/presentation/views.py
index 942f77b37abc..533d7a0c7e4a 100644
--- a/products/warehouse_suggestions/backend/presentation/views.py
+++ b/products/warehouse_suggestions/backend/presentation/views.py
@@ -37,6 +37,7 @@ from ..facade.contracts import (
Suggestion,
SuggestionAlreadyDecidedError,
SuggestionNotFoundError,
+ SuggestionOutdatedError,
SuggestionPage,
SuggestionSubjectGoneError,
)
@@ -238,5 +239,7 @@ class WarehouseSuggestionViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet
raise ValidationError({"subject_id": SUBJECT_GONE[error.subject_kind]})
except RefreshIntervalRefusedError as error:
raise ValidationError({"refresh_interval_seconds": str(error)})
+ except SuggestionOutdatedError as error:
+ raise Conflict(str(error))
except AcceptFailedError as error:
raise APIException(str(error))
diff --git a/products/warehouse_suggestions/backend/tests/test_accept.py b/products/warehouse_suggestions/backend/tests/test_accept.py
index ba4045605160..615110e5faa5 100644
--- a/products/warehouse_suggestions/backend/tests/test_accept.py
+++ b/products/warehouse_suggestions/backend/tests/test_accept.py
@@ -22,7 +22,11 @@ from products.warehouse_suggestions.backend.facade.contracts import (
SuggestionAlreadyDecidedError,
SuggestionPayload,
)
-from products.warehouse_suggestions.backend.facade.enums import WarehouseSuggestionKind, WarehouseSuggestionStatus
+from products.warehouse_suggestions.backend.facade.enums import (
+ WarehouseSuggestionKind,
+ WarehouseSuggestionRefreshMode,
+ WarehouseSuggestionStatus,
+)
from products.warehouse_suggestions.backend.logic.accept import CertificationAcceptor
from products.warehouse_suggestions.backend.logic.suggestions import HUMAN_TRANSITIONS, transition_to
from products.warehouse_suggestions.backend.models import WarehouseSuggestion
@@ -33,21 +37,34 @@ from .test_suggestions import make_draft
FLAG = "products.warehouse_suggestions.backend.presentation.views.is_warehouse_suggestions_enabled"
DAY_SECONDS = 24 * 60 * 60
REPORT = "products.warehouse_suggestions.backend.logic.analytics.report_user_or_team_action"
+DAILY_COUNTS = "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY day"
+DAILY_COUNT_COLUMNS = {
+ "day": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')"},
+ "c": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt64"},
+}
+STORED_DAY_KEY = {"enabled": True, "incremental_key": "day", "unique_key": ["day"], "lookback_seconds": 0}
+MATERIALIZE_PAYLOAD = MaterializePayload(
+ subject_name="orders",
+ refresh_interval_seconds=DAY_SECONDS,
+ saves_seconds_per_month=900,
+ saves_bytes_per_month=0,
+ freshness_today_seconds=None,
+ freshness_after_seconds=DAY_SECONDS,
+ live_sources=(),
+ unknown_sources=(),
+)
+INCREMENTAL_DAY_PAYLOAD = replace(
+ MATERIALIZE_PAYLOAD,
+ refresh_mode=WarehouseSuggestionRefreshMode.INCREMENTAL,
+ incremental_key="day",
+ unique_key=("day",),
+)
PAYLOADS: dict[WarehouseSuggestionKind, SuggestionPayload] = {
WarehouseSuggestionKind.CERTIFY: CertifyPayload(subject_name="orders"),
WarehouseSuggestionKind.DEPRECATE: DeprecatePayload(
subject_name="orders", refresh_seconds_per_month=0, refresh_bytes_per_month=0
),
- WarehouseSuggestionKind.MATERIALIZE: MaterializePayload(
- subject_name="orders",
- refresh_interval_seconds=DAY_SECONDS,
- saves_seconds_per_month=900,
- saves_bytes_per_month=0,
- freshness_today_seconds=None,
- freshness_after_seconds=DAY_SECONDS,
- live_sources=(),
- unknown_sources=(),
- ),
+ WarehouseSuggestionKind.MATERIALIZE: MATERIALIZE_PAYLOAD,
}
@@ -84,6 +101,10 @@ class TestAcceptSuggestion(APIBaseTest):
def _suggest(self, kind: WarehouseSuggestionKind) -> WarehouseSuggestion:
return suggest(self.team.id, self.view.id, kind)
+ def _suggest_materialize(self, payload: MaterializePayload) -> WarehouseSuggestion:
+ draft = make_draft(fingerprint="materialize:orders", subject_id=self.view.id)
+ return ingest_surfaced(self.team.id, replace(draft, kind=WarehouseSuggestionKind.MATERIALIZE, payload=payload))
+
def _accept(self, suggestion_id: UUID, body: dict | None = None) -> dict:
response = self.client.post(f"{self.url}/{suggestion_id}/accept/", body or {})
return {**response.json(), "http_status": response.status_code}
@@ -177,6 +198,50 @@ class TestAcceptSuggestion(APIBaseTest):
12 * 60 * 60,
)
+ @parameterized.expand(
+ [
+ (
+ "query_still_groups_by_the_key",
+ DAILY_COUNTS,
+ INCREMENTAL_DAY_PAYLOAD,
+ status.HTTP_200_OK,
+ STORED_DAY_KEY,
+ ),
+ (
+ "query_changed_since_the_suggestion",
+ "SELECT timestamp, event FROM events",
+ INCREMENTAL_DAY_PAYLOAD,
+ status.HTTP_409_CONFLICT,
+ STORED_DAY_KEY,
+ ),
+ (
+ "query_no_longer_groups",
+ "SELECT toStartOfDay(timestamp) AS day FROM events",
+ INCREMENTAL_DAY_PAYLOAD,
+ status.HTTP_409_CONFLICT,
+ STORED_DAY_KEY,
+ ),
+ ("full_refresh_clears_leftover_keys", DAILY_COUNTS, MATERIALIZE_PAYLOAD, status.HTTP_200_OK, None),
+ ]
+ )
+ def test_accepting_applies_the_refresh_mode_the_suggestion_shows(
+ self,
+ _name: str,
+ hogql: str,
+ payload: MaterializePayload,
+ expected_status: int,
+ expected_config: dict | None,
+ ) -> None:
+ DataWarehouseSavedQuery.objects.filter(id=self.view.id).update(
+ query={"kind": "HogQLQuery", "query": hogql}, columns=DAILY_COUNT_COLUMNS, incremental_config=STORED_DAY_KEY
+ )
+ suggestion = self._suggest_materialize(payload)
+
+ accepted = self._accept(suggestion.id)
+
+ self.view.refresh_from_db()
+ assert (accepted["http_status"], self.view.incremental_config) == (expected_status, expected_config), accepted
+
@parameterized.expand([("not_a_schedulable_interval", 7), ("past_the_largest_interval", 10**14)])
def test_a_refused_interval_is_a_field_error_and_keeps_the_suggestion_open(
self, _name: str, refresh_interval_seconds: int
diff --git a/products/warehouse_suggestions/backend/tests/test_candidates.py b/products/warehouse_suggestions/backend/tests/test_candidates.py
index 266de9ea0560..e2dc6e625e88 100644
--- a/products/warehouse_suggestions/backend/tests/test_candidates.py
+++ b/products/warehouse_suggestions/backend/tests/test_candidates.py
@@ -3,20 +3,27 @@ from datetime import timedelta
from uuid import uuid4
from posthog.test.base import BaseTest
+from unittest.mock import patch
from django.test import SimpleTestCase
from parameterized import parameterized
from products.data_catalog.backend.facade.enums import CertificationStatus
-from products.warehouse_suggestions.backend.facade.contracts import CertifyPayload
-from products.warehouse_suggestions.backend.facade.enums import WarehouseSuggestionAssetOutcome
+from products.data_modeling.backend.facade.models import DAG, DataWarehouseSavedQuery, Node, NodeType
+from products.warehouse_suggestions.backend.facade.contracts import CertifyPayload, MaterializePayload
+from products.warehouse_suggestions.backend.facade.enums import (
+ WarehouseSuggestionAssetOutcome,
+ WarehouseSuggestionRefreshIntervalBasis,
+ WarehouseSuggestionRefreshMode,
+)
from products.warehouse_suggestions.backend.logic.candidates.certify import CertifyCandidate
from products.warehouse_suggestions.backend.logic.candidates.deprecate import DeprecateCandidate
from products.warehouse_suggestions.backend.logic.candidates.materialize import (
MaterializeCandidate,
allowed_intervals,
estimate_savings,
+ interval_basis,
)
from products.warehouse_suggestions.backend.logic.rules import RULES, Surface
from products.warehouse_suggestions.backend.tests.factories import (
@@ -32,6 +39,12 @@ VIEW_ID = uuid4()
AT_CERTIFY_FLOORS = {"human_users": 5, "human_days": 20}
HOUR = timedelta(hours=1)
DAY = timedelta(hours=24)
+INCREMENTAL_FLAG = "products.warehouse_suggestions.backend.logic.candidates.materialize.incremental_enabled"
+DAILY_COUNTS = "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY day"
+DAILY_COUNT_COLUMNS = {
+ "day": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')"},
+ "c": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt64"},
+}
class TestCertifyCandidate(SimpleTestCase):
@@ -128,12 +141,15 @@ class TestCertifyCandidate(SimpleTestCase):
class TestMaterializeRules(SimpleTestCase):
@parameterized.expand(
[
- ("read_most_days", 25, [HOUR, timedelta(hours=6), DAY]),
- ("read_on_few_days", 10, [DAY]),
+ ("read_every_hour_on_most_days", 25, 25 * 24, [HOUR, timedelta(hours=6), DAY]),
+ ("read_a_few_times_a_day", 25, 25 * 3, [DAY]),
+ ("read_on_few_days", 10, 10 * 24, [DAY]),
]
)
- def test_read_cadence_sets_the_shortest_interval(self, _name: str, human_days: int, expected: list) -> None:
- reads = busy_reads(human_days=human_days)
+ def test_read_cadence_sets_the_shortest_interval(
+ self, _name: str, human_days: int, human_requests: int, expected: list
+ ) -> None:
+ reads = busy_reads(human_days=human_days, human_requests=human_requests)
intervals = allowed_intervals(
context(team_reads({})), [HOUR, timedelta(hours=6), DAY, timedelta(days=7)], reads
@@ -157,18 +173,108 @@ class TestMaterializeRules(SimpleTestCase):
@parameterized.expand(
[
- ("limited", "SELECT timestamp, event FROM events ORDER BY timestamp DESC LIMIT 10"),
- ("no_key_column", "SELECT count() FROM events"),
+ (
+ "sources_allow_nothing_finer",
+ HOUR,
+ [HOUR, DAY],
+ True,
+ WarehouseSuggestionRefreshIntervalBasis.SOURCE_SYNC,
+ ),
+ (
+ "nothing_refreshes_faster",
+ HOUR,
+ [HOUR, DAY],
+ False,
+ WarehouseSuggestionRefreshIntervalBasis.FASTEST_SCHEDULE,
+ ),
+ ("reads_allow_nothing_finer", DAY, [DAY], False, WarehouseSuggestionRefreshIntervalBasis.READ_RATE),
+ (
+ "finer_ones_save_too_little",
+ DAY,
+ [timedelta(hours=6), DAY],
+ False,
+ WarehouseSuggestionRefreshIntervalBasis.SAVINGS,
+ ),
]
)
- def test_a_view_that_cannot_refresh_incrementally_is_not_proposed(self, _name: str, hogql: str) -> None:
- reads = team_reads({view_subject(VIEW_ID): busy_reads()})
- not_incremental = view(VIEW_ID, hogql=hogql)
+ def test_interval_basis_names_what_kept_the_interval_from_being_shorter(
+ self,
+ _name: str,
+ interval: timedelta,
+ read_allowed: list[timedelta],
+ limited_by_sources: bool,
+ expected: WarehouseSuggestionRefreshIntervalBasis,
+ ) -> None:
+ basis = interval_basis(
+ interval, [HOUR, timedelta(hours=6), DAY], read_allowed, limited_by_sources=limited_by_sources
+ )
+
+ assert basis == expected
- result = MaterializeCandidate().evaluate(context(reads, views=[not_incremental]))
- assert result.drafts == ()
- assert [rejection.reason for rejection in result.rejections] == ["its query cannot refresh incrementally"]
+class TestMaterializeRefreshMode(BaseTest):
+ @parameterized.expand(
+ [
+ ("grouped_by_day", DAILY_COUNTS, True, WarehouseSuggestionRefreshMode.INCREMENTAL, "day", ("day",)),
+ ("flag_off", DAILY_COUNTS, False, WarehouseSuggestionRefreshMode.FULL, None, ()),
+ (
+ "limited_query",
+ "SELECT timestamp, event FROM events ORDER BY timestamp DESC LIMIT 10",
+ True,
+ WarehouseSuggestionRefreshMode.FULL,
+ None,
+ (),
+ ),
+ (
+ "grouped_over_a_nondeterministic_view",
+ DAILY_COUNTS.replace("FROM events", "FROM sampled_events"),
+ True,
+ WarehouseSuggestionRefreshMode.FULL,
+ None,
+ (),
+ ),
+ ]
+ )
+ def test_a_suggestion_proposes_incremental_only_when_the_keys_are_certain(
+ self,
+ _name: str,
+ hogql: str,
+ flag_on: bool,
+ refresh_mode: WarehouseSuggestionRefreshMode,
+ incremental_key: str | None,
+ unique_key: tuple[str, ...],
+ ) -> None:
+ DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="sampled_events",
+ query={"kind": "HogQLQuery", "query": "SELECT timestamp FROM events WHERE rand() % 10 = 0"},
+ columns={"timestamp": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')"}},
+ )
+ saved_query = DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="daily_counts",
+ query={"kind": "HogQLQuery", "query": hogql},
+ columns=DAILY_COUNT_COLUMNS,
+ )
+ dag = DAG.objects.create(team=self.team, name="Default")
+ Node.objects.create(team=self.team, dag=dag, name="daily_counts", saved_query=saved_query, type=NodeType.VIEW)
+ candidate_context = context(
+ team_reads({view_subject(saved_query.id): busy_reads(human_duration_ms=200 * 10_000)}),
+ views=[view(saved_query.id, name="daily_counts", hogql=hogql)],
+ team_id=self.team.pk,
+ )
+
+ with patch(INCREMENTAL_FLAG, return_value=flag_on):
+ [draft] = MaterializeCandidate().evaluate(candidate_context).drafts
+
+ assert isinstance(draft.payload, MaterializePayload)
+ expected_lookback = RULES.materialize.incremental_lookback_seconds if incremental_key else 0
+ assert (
+ draft.payload.refresh_mode,
+ draft.payload.incremental_key,
+ draft.payload.unique_key,
+ draft.payload.lookback_seconds,
+ ) == (refresh_mode, incremental_key, unique_key, expected_lookback)
class TestDeprecateCandidate(SimpleTestCase):
diff --git a/products/warehouse_suggestions/backend/tests/test_job.py b/products/warehouse_suggestions/backend/tests/test_job.py
index 60078f8ebf89..c32bfcd38e54 100644
--- a/products/warehouse_suggestions/backend/tests/test_job.py
+++ b/products/warehouse_suggestions/backend/tests/test_job.py
@@ -2,6 +2,7 @@ from datetime import UTC, date, datetime, time, timedelta
from uuid import uuid4
from posthog.test.base import BaseTest, ClickhouseTestMixin
+from unittest.mock import patch
from parameterized import parameterized
@@ -22,6 +23,12 @@ TODAY = date.today()
NOW = datetime.combine(TODAY, time(hour=9), tzinfo=UTC)
BEFORE_THE_WINDOW = NOW - timedelta(days=2 * RULES.window_days)
SCENES = ("SQLEditor", "Dashboard")
+INCREMENTAL_FLAG = "products.warehouse_suggestions.backend.logic.candidates.materialize.incremental_enabled"
+DAILY_COUNTS = "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY day"
+DAILY_COUNT_COLUMNS = {
+ "day": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')"},
+ "c": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt64"},
+}
FULL_ROLLUP = RollupDays(days_with_data=RULES.window_days, recent_days_with_data=RULES.lifecycle.expire_after_days)
@@ -47,6 +54,9 @@ class TestRunTeam(ClickhouseTestMixin, BaseTest):
def test_proposes_each_kind_from_a_month_of_reads_and_surfaces_them(self) -> None:
dag = DAG.objects.create(team=self.team, name="Default")
busy, busy_node = self._view("busy_orders", dag)
+ DataWarehouseSavedQuery.objects.filter(id=busy.id).update(
+ query={"kind": "HogQLQuery", "query": DAILY_COUNTS}, columns=DAILY_COUNT_COLUMNS
+ )
for name, properties in [("events", {"origin": "posthog"}), ("legacy_orders", {})]:
table = Node.objects.create(team=self.team, dag=dag, name=name, type=NodeType.TABLE, properties=properties)
Edge.objects.create(team=self.team, dag=dag, source=table, target=busy_node)
@@ -72,7 +82,8 @@ class TestRunTeam(ClickhouseTestMixin, BaseTest):
],
)
- result = run_team(self.team.pk, run_id="run-1", today=TODAY, rollup_days=FULL_ROLLUP, now=NOW)
+ with patch(INCREMENTAL_FLAG, return_value=True):
+ result = run_team(self.team.pk, run_id="run-1", today=TODAY, rollup_days=FULL_ROLLUP, now=NOW)
assert result.status == TeamRunStatus.PROCESSED
assert sorted(
@@ -93,6 +104,11 @@ class TestRunTeam(ClickhouseTestMixin, BaseTest):
materialize = WarehouseSuggestion.objects.for_team(self.team.pk).get(kind=WarehouseSuggestionKind.MATERIALIZE)
assert (materialize.payload["refresh_interval_seconds"], materialize.run_id) == (24 * 60 * 60, "run-1")
assert [source["name"] for source in materialize.payload["live_sources"]] == ["events"]
+ assert (
+ materialize.payload["refresh_mode"],
+ materialize.payload["incremental_key"],
+ materialize.payload["unique_key"],
+ ) == ("incremental", "day", ["day"])
@parameterized.expand(
[
diff --git a/products/warehouse_suggestions/frontend/generated/api.schemas.ts b/products/warehouse_suggestions/frontend/generated/api.schemas.ts
index 7aadd203da18..0037b6714563 100644
--- a/products/warehouse_suggestions/frontend/generated/api.schemas.ts
+++ b/products/warehouse_suggestions/frontend/generated/api.schemas.ts
@@ -28,11 +28,51 @@ export interface WarehouseSuggestionVisibleSourcesApi {
hidden_count: number
}
+/**
+ * * `full` - Full
+ * * `incremental` - Incremental
+ */
+export type WarehouseSuggestionRefreshModeEnumApi =
+ (typeof WarehouseSuggestionRefreshModeEnumApi)[keyof typeof WarehouseSuggestionRefreshModeEnumApi]
+
+export const WarehouseSuggestionRefreshModeEnumApi = {
+ Full: 'full',
+ Incremental: 'incremental',
+} as const
+
+/**
+ * * `source_sync` - Source sync
+ * * `fastest_schedule` - Fastest schedule
+ * * `read_rate` - Read rate
+ * * `savings` - Savings
+ */
+export type WarehouseSuggestionRefreshIntervalBasisEnumApi =
+ (typeof WarehouseSuggestionRefreshIntervalBasisEnumApi)[keyof typeof WarehouseSuggestionRefreshIntervalBasisEnumApi]
+
+export const WarehouseSuggestionRefreshIntervalBasisEnumApi = {
+ SourceSync: 'source_sync',
+ FastestSchedule: 'fastest_schedule',
+ ReadRate: 'read_rate',
+ Savings: 'savings',
+} as const
+
export interface WarehouseSuggestionMaterializePayloadApi {
/** Sources that are always current, such as PostHog tables and direct connections. */
live_sources: WarehouseSuggestionVisibleSourcesApi
/** Sources with no sync schedule, so their freshness is unknown. */
unknown_sources: WarehouseSuggestionVisibleSourcesApi
+ /** incremental: each refresh reads only rows with a new incremental_key value and updates rows by unique_key. full: each refresh rebuilds the whole table. Incremental is proposed only when the keys are certain from the query.
+ *
+ * * `full` - Full
+ * * `incremental` - Incremental */
+ refresh_mode: WarehouseSuggestionRefreshModeEnumApi
+ /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.
+ *
+ * * `source_sync` - Source sync
+ * * `fastest_schedule` - Fastest schedule
+ * * `read_rate` - Read rate
+ * * `savings` - Savings */
+ refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasisEnumApi | null
/** Name of the view to materialize. */
subject_name: string
/** Proposed refresh interval, in seconds. */
@@ -48,6 +88,15 @@ export interface WarehouseSuggestionMaterializePayloadApi {
freshness_today_seconds: number | null
/** How old the data can be once materialized, in seconds. */
freshness_after_seconds: number
+ /**
+ * Date or datetime output column whose advancing value marks new rows. Null for a full refresh.
+ * @nullable
+ */
+ incremental_key?: string | null
+ /** Output columns that identify a row, so a refresh updates rows instead of adding duplicates. Empty for a full refresh. */
+ unique_key?: string[]
+ /** How far before the last refresh's highest incremental_key value each refresh reads again, in seconds. The last incremental_key value is always read again. */
+ lookback_seconds?: number
}
export type WarehouseSuggestionPayloadApi =
diff --git a/products/warehouse_suggestions/mcp/tools.yaml b/products/warehouse_suggestions/mcp/tools.yaml
index 9218ef366f35..c90ec63771ed 100644
--- a/products/warehouse_suggestions/mcp/tools.yaml
+++ b/products/warehouse_suggestions/mcp/tools.yaml
@@ -24,7 +24,9 @@ tools:
Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as
certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on
the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another
- schedule. Requires human typed confirmation.
+ schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by
+ incremental_key and update rows by unique_key, full refreshes rebuild the whole table. A 409 means the view
+ changed since the suggestion was made and its keys no longer apply. Requires human typed confirmation.
confirmed_action:
message: >
About to accept warehouse suggestion '{id}'. This applies the change it proposes to the view or table.
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index 136d53f8d150..d79562411ea2 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -14752,7 +14752,7 @@
}
},
"warehouse-suggestions-accept-execute": {
- "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. Requires human typed confirmation.",
+ "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. A 409 means the view changed since the suggestion was made and its keys no longer apply. Requires human typed confirmation.",
"category": "Warehouse suggestions",
"feature": "warehouse_suggestions",
"summary": "Accept a warehouse suggestion (execute)",
@@ -14767,7 +14767,7 @@
"feature_flag": "warehouse-suggestions"
},
"warehouse-suggestions-accept-prepare": {
- "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. Requires human typed confirmation.",
+ "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. A 409 means the view changed since the suggestion was made and its keys no longer apply. Requires human typed confirmation.",
"category": "Warehouse suggestions",
"feature": "warehouse_suggestions",
"summary": "Accept a warehouse suggestion (prepare)",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index a20cce51ece9..3a345f1518a6 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -15446,7 +15446,7 @@
}
},
"warehouse-suggestions-accept-execute": {
- "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. Requires human typed confirmation.",
+ "description": "Step 2 of 2 for accept warehouse suggestion. Verifies the confirmation_hash from -prepare and the literal \"confirm\" string typed by the user, then performs the action. ONLY call this after the user has explicitly typed \"confirm\" in chat. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. A 409 means the view changed since the suggestion was made and its keys no longer apply. Requires human typed confirmation.",
"category": "Warehouse suggestions",
"feature": "warehouse_suggestions",
"summary": "Accept a warehouse suggestion (execute)",
@@ -15461,7 +15461,7 @@
"feature_flag": "warehouse-suggestions"
},
"warehouse-suggestions-accept-prepare": {
- "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. Requires human typed confirmation.",
+ "description": "Step 1 of 2 for accept warehouse suggestion. Validates the arguments and returns a signed confirmation_hash plus a message to surface to the user. The user must reply with the literal word \"confirm\" before you call the matching -execute tool with the hash. Original action: Accept a proposed warehouse suggestion. Certify and deprecate suggestions mark the view or table as certified or deprecated in the data catalog. Materialize suggestions turn on materialization for the view on the proposed schedule, which adds a recurring compute cost; pass refresh_interval_seconds to use another schedule. They also apply the payload's refresh_mode: incremental refreshes read only new rows by incremental_key and update rows by unique_key, full refreshes rebuild the whole table. A 409 means the view changed since the suggestion was made and its keys no longer apply. Requires human typed confirmation.",
"category": "Warehouse suggestions",
"feature": "warehouse_suggestions",
"summary": "Accept a warehouse suggestion (prepare)",
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 2063167f66f7..58933b75857e 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -78918,11 +78918,51 @@ export namespace Schemas {
hidden_count: number;
}
+ /**
+ * * `full` - Full
+ * * `incremental` - Incremental
+ */
+ export type WarehouseSuggestionRefreshModeEnum = typeof WarehouseSuggestionRefreshModeEnum[keyof typeof WarehouseSuggestionRefreshModeEnum];
+
+
+ export const WarehouseSuggestionRefreshModeEnum = {
+ Full: 'full',
+ Incremental: 'incremental',
+ } as const;
+
+ /**
+ * * `source_sync` - Source sync
+ * * `fastest_schedule` - Fastest schedule
+ * * `read_rate` - Read rate
+ * * `savings` - Savings
+ */
+ export type WarehouseSuggestionRefreshIntervalBasisEnum = typeof WarehouseSuggestionRefreshIntervalBasisEnum[keyof typeof WarehouseSuggestionRefreshIntervalBasisEnum];
+
+
+ export const WarehouseSuggestionRefreshIntervalBasisEnum = {
+ SourceSync: 'source_sync',
+ FastestSchedule: 'fastest_schedule',
+ ReadRate: 'read_rate',
+ Savings: 'savings',
+ } as const;
+
export interface WarehouseSuggestionMaterializePayload {
/** Sources that are always current, such as PostHog tables and direct connections. */
live_sources: WarehouseSuggestionVisibleSources;
/** Sources with no sync schedule, so their freshness is unknown. */
unknown_sources: WarehouseSuggestionVisibleSources;
+ /** incremental: each refresh reads only rows with a new incremental_key value and updates rows by unique_key. full: each refresh rebuilds the whole table. Incremental is proposed only when the keys are certain from the query.
+ *
+ * * `full` - Full
+ * * `incremental` - Incremental */
+ refresh_mode: WarehouseSuggestionRefreshModeEnum;
+ /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.
+ *
+ * * `source_sync` - Source sync
+ * * `fastest_schedule` - Fastest schedule
+ * * `read_rate` - Read rate
+ * * `savings` - Savings */
+ refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasisEnum | null;
/** Name of the view to materialize. */
subject_name: string;
/** Proposed refresh interval, in seconds. */
@@ -78938,6 +78978,15 @@ export namespace Schemas {
freshness_today_seconds: number | null;
/** How old the data can be once materialized, in seconds. */
freshness_after_seconds: number;
+ /**
+ * Date or datetime output column whose advancing value marks new rows. Null for a full refresh.
+ * @nullable
+ */
+ incremental_key?: string | null;
+ /** Output columns that identify a row, so a refresh updates rows instead of adding duplicates. Empty for a full refresh. */
+ unique_key?: string[];
+ /** How far before the last refresh's highest incremental_key value each refresh reads again, in seconds. The last incremental_key value is always read again. */
+ lookback_seconds?: number;
}
export type WarehouseSuggestionPayload = WarehouseSuggestionCertifyPayload | WarehouseSuggestionDeprecatePayload | WarehouseSuggestionMaterializePayload;