:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-suggestions): suggest materializing views that refreshes recompute (#124125)
Thiago Salvatore7271fe706696
diff --git a/products/warehouse_suggestions/backend/facade/contracts.py b/products/warehouse_suggestions/backend/facade/contracts.pyindex b8c5459de2d6..08c6a09b002c 100644--- a/products/warehouse_suggestions/backend/facade/contracts.py+++ b/products/warehouse_suggestions/backend/facade/contracts.py@@ -60,6 +60,7 @@ class MaterializePayload:unique_key: tuple[str, ...] = ()lookback_seconds: int = 0refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasis | None = None+ recomputed_by_views: int = 0@frozen@@ -99,6 +100,7 @@ class MaterializeSuggestionPayload:unique_key: tuple[str, ...] = ()lookback_seconds: int = 0refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasis | None = None+ recomputed_by_views: int = 0SuggestionPayloadView = CertifyPayload | DeprecatePayload | MaterializeSuggestionPayload | UnmaterializePayloaddiff --git a/products/warehouse_suggestions/backend/facade/enums.py b/products/warehouse_suggestions/backend/facade/enums.pyindex 25a8ba1a8002..ad05c93ac071 100644--- a/products/warehouse_suggestions/backend/facade/enums.py+++ b/products/warehouse_suggestions/backend/facade/enums.py@@ -44,4 +44,5 @@ class WarehouseSuggestionRefreshIntervalBasis(LabeledStrEnum):SOURCE_SYNC = ("source_sync", "Source sync")FASTEST_SCHEDULE = ("fastest_schedule", "Fastest schedule")READ_RATE = ("read_rate", "Read rate")+ CONSUMER_REFRESH = ("consumer_refresh", "Consumer refresh")SAVINGS = "savings"diff --git a/products/warehouse_suggestions/backend/logic/candidates/base.py b/products/warehouse_suggestions/backend/logic/candidates/base.pyindex 812c28f50f0d..cbd4580f90f3 100644--- a/products/warehouse_suggestions/backend/logic/candidates/base.py+++ b/products/warehouse_suggestions/backend/logic/candidates/base.py@@ -155,6 +155,7 @@ class CandidateResult:drafts: tuple[SuggestionDraft, ...]rejections: tuple[Rejection, ...]skipped_reason: str | None = None+ measurement_requests: tuple[Subject, ...] = ()class Candidate(ABC):diff --git a/products/warehouse_suggestions/backend/logic/candidates/materialize.py b/products/warehouse_suggestions/backend/logic/candidates/materialize.pyindex 9b8da6b6ca2e..6298253bc8e1 100644--- a/products/warehouse_suggestions/backend/logic/candidates/materialize.py+++ b/products/warehouse_suggestions/backend/logic/candidates/materialize.py@@ -14,6 +14,7 @@ from products.data_modeling.backend.facade.api import (MissingDagNodeError,SavedQueryFrequencyBounds,definition_database,+ dependent_saved_query_ids,get_saved_query_columns,propose_incremental_config,saved_query_target_bounds,@@ -29,7 +30,10 @@ from ...facade.enums import (WarehouseSuggestionRefreshMode,WarehouseSuggestionSubjectKind,)+from ..lineage import recomputing_consumersfrom ..reads import Subject, SubjectReads+from ..rules import MaterializeRules+from ..view_costs import unfinished_measurementfrom .base import (DAYS_PER_MONTH,MILLISECONDS_PER_SECOND,@@ -43,11 +47,14 @@ from .base import (savings_between,stored_cost,)+from .unmaterialize import effective_definitionSECONDS_PER_MONTH = DAYS_PER_MONTH * 24 * 60 * 60READ_SECONDS_PER_MONTH = "read_seconds_per_month"READ_BYTES_PER_MONTH = "read_bytes_per_month"HUMAN_READS_PER_MONTH = "human_reads_per_month"+UNMEASURED_REJECTION = "its cost has not been measured yet"+NOT_RECOMPUTED_ENOUGH_REJECTION = "it is not computed often enough for a stored copy to save half of it"@frozen@@ -73,6 +80,55 @@ class _Proposal:estimate: SavingsEstimate+@frozen+class Recomputations:+ consumer_runs_per_month: float+ consumers: int+ fastest_consumer_interval: timedelta+++@frozen+class ReadCosts:+ observed: QueryCost+ saved: QueryCost+++@frozen+class _Unmeasured:+ recomputations: Recomputations+++class _Consumers:+ def __init__(self, context: CandidateContext) -> None:+ self._context = context++ @cached_property+ def _materialized_ids(self) -> frozenset[UUID]:+ return frozenset(+ saved_query.id for saved_query in self._context.inventory.saved_queries.values() if saved_query.materializes+ )++ @cached_property+ def _refreshed_ids(self) -> frozenset[UUID]:+ return frozenset(+ saved_query_id+ for saved_query_id in self._materialized_ids+ if (refreshes := self._context.reads.refreshes.get(saved_query_id)) is not None+ and refreshes.runs > 0+ and not self._context.inventory.saved_queries[saved_query_id].refreshes_incrementally+ )++ @cached_property+ def _dependents(self) -> Mapping[UUID, frozenset[UUID]]:+ return dependent_saved_query_ids(self._context.team_id, list(self._context.inventory.saved_queries))++ def refreshing(self, saved_query_id: UUID) -> frozenset[UUID]:+ if not self._refreshed_ids:+ return frozenset()+ consumers = recomputing_consumers(saved_query_id, self._dependents, self._materialized_ids)+ return consumers & self._refreshed_ids++class _IncrementalProposer:"""Proposes incremental keys for one team run, building the team database at most once."""@@ -100,33 +156,45 @@ class MaterializeCandidate(Candidate):def evaluate(self, context: CandidateContext) -> CandidateResult:drafts = []rejections = []+ unmeasured: list[tuple[Subject, Recomputations]] = []incremental = _IncrementalProposer(context.team_id)+ consumers = _Consumers(context)for saved_query in context.inventory.saved_queries.values():subject = Subject(kind=WarehouseSuggestionSubjectKind.SAVED_QUERY, id=saved_query.id)+ if not self._is_unmaterialized(context, saved_query):+ continuereads = context.reads.reads_of(subject)- if reads is None or reads.human_requests == 0 or not self._is_unmaterialized(context, saved_query):+ consumer_ids = consumers.refreshing(saved_query.id)+ if consumer_ids:+ outcome = self._propose_recomputed(context, saved_query, reads, consumer_ids, incremental)+ elif reads is None or reads.human_requests == 0:+ continue+ else:+ outcome = self._propose(context, saved_query, reads, incremental)+ if isinstance(outcome, _Unmeasured):+ unmeasured.append((subject, outcome.recomputations))+ rejections.append(Rejection(subject=subject, reason=UNMEASURED_REJECTION))continue- outcome = self._propose(context, saved_query, reads, incremental)if isinstance(outcome, str):rejections.append(Rejection(subject=subject, reason=outcome))continue+ score_inputs = {+ "seconds_saved_per_month": outcome.estimate.seconds_saved,+ "bytes_saved_per_month": outcome.estimate.bytes_saved,+ READ_SECONDS_PER_MONTH: outcome.estimate.read_seconds,+ READ_BYTES_PER_MONTH: outcome.estimate.read_bytes,+ "refresh_interval_seconds": outcome.estimate.interval.total_seconds(),+ }+ if not consumer_ids and reads is not None:+ score_inputs[HUMAN_READS_PER_MONTH] = reads.human_reads * context.month_scaledrafts.append(- self.draft(- context,- subject,- outcome.payload,- score=outcome.estimate.score,- score_inputs={- "seconds_saved_per_month": outcome.estimate.seconds_saved,- "bytes_saved_per_month": outcome.estimate.bytes_saved,- READ_SECONDS_PER_MONTH: outcome.estimate.read_seconds,- READ_BYTES_PER_MONTH: outcome.estimate.read_bytes,- HUMAN_READS_PER_MONTH: reads.human_reads * context.month_scale,- "refresh_interval_seconds": outcome.estimate.interval.total_seconds(),- },- )+ self.draft(context, subject, outcome.payload, score=outcome.estimate.score, score_inputs=score_inputs))- return CandidateResult(drafts=tuple(drafts), rejections=tuple(rejections))+ return CandidateResult(+ drafts=tuple(drafts),+ rejections=tuple(rejections),+ measurement_requests=_most_recomputed(unmeasured, context.rules.measurement.max_per_team),+ )def asset_outcome(self, context: CandidateContext, subject: Subject, accepted_at: datetime | None@@ -221,6 +289,63 @@ class MaterializeCandidate(Candidate):estimate=estimate,)+ def _propose_recomputed(+ self,+ context: CandidateContext,+ saved_query: SavedQueryDefinition,+ reads: SubjectReads | None,+ consumer_ids: frozenset[UUID],+ incremental: _IncrementalProposer,+ ) -> _Proposal | _Unmeasured | str:+ if context.accepted_recently(WarehouseSuggestionKind.UNMATERIALIZE, saved_query.id):+ return f"was unmaterialized on a suggestion in the last {context.rules.lifecycle.flip_cooldown_days} days"+ recomputations = count_recomputations(context, consumer_ids)+ ceiling = min(context.rules.refresh.max_interval, recomputations.fastest_consumer_interval)+ reads_qualify = (+ reads is not None and reads.human_requests > 0 and self._read_floor_rejection(context, reads) is None+ )+ if not reads_qualify and not recomputed_often_enough(context, recomputations, ceiling):+ return NOT_RECOMPUTED_ENOUGH_REJECTION+ frequency = saved_query_target_bounds(context.team_id, saved_query.id)+ if frequency is None:+ capture_exception(MissingDagNodeError(f"Saved query {saved_query.id} has no DAG node"))+ return "it has no node in a DAG"+ intervals = sorted(interval for interval in frequency.bounds.allowed if interval <= ceiling)+ if not intervals:+ return "no refresh interval keeps up with the views that read it"+ interval = intervals[-1]+ if not reads_qualify and not recomputed_often_enough(context, recomputations, interval):+ return NOT_RECOMPUTED_ENOUGH_REJECTION+ cost = evaluation_cost(context, saved_query, reads)+ if isinstance(cost, str):+ return cost+ if cost is None:+ return _Unmeasured(recomputations=recomputations)+ estimate = recomputation_savings(context, reads, recomputations, interval, cost)+ if not estimate.clears_a_floor:+ return "savings stay under the floors at the interval the views that read it need"+ sources = _classify_sources(context, saved_query.id, frequency)+ freshness_today = int(frequency.max_data_age.total_seconds()) or None+ interval_seconds = int(interval.total_seconds())+ payload = MaterializePayload(+ subject_name=saved_query.name,+ refresh_interval_seconds=interval_seconds,+ saves_seconds_per_month=estimate.seconds_saved,+ saves_bytes_per_month=estimate.bytes_saved,+ freshness_today_seconds=freshness_today,+ freshness_after_seconds=interval_seconds + (freshness_today or 0),+ live_sources=sources.live,+ unknown_sources=sources.unknown,+ refresh_interval_basis=WarehouseSuggestionRefreshIntervalBasis.CONSUMER_REFRESH,+ recomputed_by_views=recomputations.consumers,+ )+ return _Proposal(+ payload=_with_refresh_mode(+ payload, incremental.config_for(saved_query), context.rules.materialize.incremental_lookback_seconds+ ),+ estimate=estimate,+ )+def _read_floor_rejection(self, context: CandidateContext, reads: SubjectReads) -> str | None:rules = context.rules.materializemin_days = context.scaled_floor(rules.min_days)@@ -290,30 +415,121 @@ def refresh_cost(context: CandidateContext, saved_query_id: UUID, reads: Subjectdef estimate_savings(context: CandidateContext, reads: SubjectReads, interval: timedelta, refresh: QueryCost) -> SavingsEstimate:- rules = context.rules.materialize+ refreshes_per_month = SECONDS_PER_MONTH / interval.total_seconds()+ read_costs = read_costs_of(context, reads)+ return _estimate(+ context.rules.materialize,+ interval,+ cost_without=read_costs.observed,+ saved=QueryCost(+ seconds=read_costs.saved.seconds - refreshes_per_month * refresh.seconds,+ read_bytes=read_costs.saved.read_bytes - refreshes_per_month * refresh.read_bytes,+ ),+ )+++def read_costs_of(context: CandidateContext, reads: SubjectReads | None) -> ReadCosts:+ if reads is None:+ return ReadCosts(observed=QueryCost(seconds=0, read_bytes=0), saved=QueryCost(seconds=0, read_bytes=0))scale = context.month_scale+ observed = QueryCost(+ seconds=reads.human_duration_ms / MILLISECONDS_PER_SECOND * scale, read_bytes=reads.human_read_bytes * scale+ )+ computed_on_read = QueryCost(+ seconds=reads.human_reads * reads.alone_duration_ms_median / MILLISECONDS_PER_SECOND * scale,+ read_bytes=reads.human_reads * reads.alone_read_bytes_median * scale,+ )+ return ReadCosts(+ observed=observed,+ saved=QueryCost(+ seconds=min(observed.seconds, computed_on_read.seconds),+ read_bytes=min(observed.read_bytes, computed_on_read.read_bytes),+ ),+ )+++def count_recomputations(context: CandidateContext, consumer_ids: frozenset[UUID]) -> Recomputations:+ consumer_runs = [context.reads.refreshes[consumer_id].runs for consumer_id in consumer_ids]+ return Recomputations(+ consumer_runs_per_month=sum(consumer_runs) * context.month_scale,+ consumers=len(consumer_ids),+ fastest_consumer_interval=timedelta(days=context.days_observed) / max(consumer_runs),+ )+++def recomputed_often_enough(context: CandidateContext, recomputations: Recomputations, interval: timedelta) -> bool:refreshes_per_month = SECONDS_PER_MONTH / interval.total_seconds()- body_seconds = reads.alone_duration_ms_median / MILLISECONDS_PER_SECOND- read_seconds = reads.human_duration_ms / MILLISECONDS_PER_SECOND * scale- read_bytes = reads.human_read_bytes * scale- seconds_saved = min(read_seconds, reads.human_reads * body_seconds * scale) - refreshes_per_month * refresh.seconds- bytes_saved = (- min(read_bytes, reads.human_reads * reads.alone_read_bytes_median * scale)- - refreshes_per_month * refresh.read_bytes+ runs = recomputations.consumer_runs_per_month+ return runs - refreshes_per_month >= context.rules.materialize.min_saved_share * runs+++def evaluation_cost(+ context: CandidateContext, saved_query: SavedQueryDefinition, reads: SubjectReads | None+) -> QueryCost | str | None:+ definition = effective_definition(context.team_id, saved_query)+ if reads is not None and reads.last_measured_at is not None:+ if definition.changed_at is None or definition.changed_at < reads.last_measured_at:+ return QueryCost(+ seconds=reads.measured_duration_ms / MILLISECONDS_PER_SECOND, read_bytes=reads.measured_read_bytes+ )+ unfinished = unfinished_measurement(context.team_id, saved_query.id)+ if unfinished is None or unfinished.fingerprint != definition.fingerprint:+ return None+ if unfinished.failed:+ return "its cost could not be measured"+ return QueryCost(seconds=unfinished.seconds, read_bytes=0)+++def recomputation_savings(+ context: CandidateContext,+ reads: SubjectReads | None,+ recomputations: Recomputations,+ interval: timedelta,+ cost: QueryCost,+) -> SavingsEstimate:+ runs = recomputations.consumer_runs_per_month+ runs_saved = runs - SECONDS_PER_MONTH / interval.total_seconds()+ read_costs = read_costs_of(context, reads)+ return _estimate(+ context.rules.materialize,+ interval,+ cost_without=QueryCost(+ seconds=runs * cost.seconds + read_costs.observed.seconds,+ read_bytes=runs * cost.read_bytes + read_costs.observed.read_bytes,+ ),+ saved=QueryCost(+ seconds=runs_saved * cost.seconds + read_costs.saved.seconds,+ read_bytes=runs_saved * cost.read_bytes + read_costs.saved.read_bytes,+ ),+ )+++def _estimate(+ rules: MaterializeRules, interval: timedelta, *, cost_without: QueryCost, saved: QueryCost+) -> SavingsEstimate:+ clears_time = (+ saved.seconds >= rules.min_seconds_saved and saved.seconds >= rules.min_saved_share * cost_without.seconds+ )+ clears_bytes = (+ saved.read_bytes >= rules.min_bytes_saved+ and saved.read_bytes >= rules.min_saved_share * cost_without.read_bytes)- clears_time = seconds_saved >= rules.min_seconds_saved and seconds_saved >= rules.min_saved_share * read_seconds- clears_bytes = bytes_saved >= rules.min_bytes_saved and bytes_saved >= rules.min_saved_share * read_bytesreturn SavingsEstimate(interval=interval,- read_seconds=read_seconds,- read_bytes=read_bytes,- seconds_saved=seconds_saved,- bytes_saved=bytes_saved,+ read_seconds=cost_without.seconds,+ read_bytes=cost_without.read_bytes,+ seconds_saved=saved.seconds,+ bytes_saved=saved.read_bytes,clears_a_floor=clears_time or clears_bytes,- score=max(seconds_saved / rules.min_seconds_saved, bytes_saved / rules.min_bytes_saved),+ score=max(saved.seconds / rules.min_seconds_saved, saved.read_bytes / rules.min_bytes_saved),)+def _most_recomputed(unmeasured: list[tuple[Subject, Recomputations]], limit: int) -> tuple[Subject, ...]:+ ranked = sorted(unmeasured, key=lambda pair: pair[1].consumer_runs_per_month, reverse=True)+ return tuple(subject for subject, _ in ranked[:limit])++def _read_cost_before_at(score_inputs: Mapping[str, float], reads_per_month: float) -> QueryCost | None:cost_before = stored_cost(score_inputs, READ_SECONDS_PER_MONTH, READ_BYTES_PER_MONTH)reads_before = score_inputs.get(HUMAN_READS_PER_MONTH)diff --git a/products/warehouse_suggestions/backend/logic/job.py b/products/warehouse_suggestions/backend/logic/job.pyindex ddf4f05e25b7..4df998cd425f 100644--- a/products/warehouse_suggestions/backend/logic/job.py+++ b/products/warehouse_suggestions/backend/logic/job.py@@ -10,7 +10,6 @@ from posthog.dataclasses import frozenfrom posthog.models.team import Teamfrom posthog.models.team.extensions import get_or_create_team_extension-from ..facade.contracts import SuggestionDraftfrom ..facade.enums import WarehouseSuggestionKind, WarehouseSuggestionStatusfrom ..models import WarehouseSuggestion, WarehouseSuggestionTeamConfigfrom .analytics import report_candidates@@ -18,7 +17,7 @@ from .candidates.base import CandidateContext, CandidateResultfrom .candidates.registry import evaluate_candidatesfrom .inventory import load_inventoryfrom .lifecycle import FLIP_KINDS, LifecycleResult, apply_run-from .reads import ReadWindow, RollupDays, TeamReads, read_rollup_days, read_team_reads+from .reads import ReadWindow, RollupDays, Subject, TeamReads, read_rollup_days, read_team_readsfrom .rules import RULES, EligibilityRules, RulesMAINTAINED_STATUSES = (WarehouseSuggestionStatus.PROPOSED, WarehouseSuggestionStatus.ACCEPTED)@@ -36,6 +35,7 @@ class TeamRunResult:status: TeamRunStatusdrafts: int = 0lifecycle: LifecycleResult | None = None+ measurement_requests: tuple[Subject, ...] = ()def run_team(@@ -59,12 +59,19 @@ def run_team(_record_run(_locked_config(team_id), reads, eligible=False, now=now)return TeamRunResult(team_id=team_id, status=status)context = build_context(team, reads, run_id=run_id, rules=rules, now=now)- drafts = _evaluate_and_report(context) if eligible else []+ results = _evaluate_and_report(context) if eligible else {}+ drafts = [draft for result in results.values() for draft in result.drafts]with transaction.atomic():config = _locked_config(team_id)_record_run(config, reads, eligible=eligible, now=now)lifecycle = apply_run(context, team, drafts, now, surface=eligible and config.paused_reason is None)- return TeamRunResult(team_id=team_id, status=status, drafts=len(drafts), lifecycle=lifecycle)+ return TeamRunResult(+ team_id=team_id,+ status=status,+ drafts=len(drafts),+ lifecycle=lifecycle,+ measurement_requests=tuple(subject for result in results.values() for subject in result.measurement_requests),+ )def explain_team(@@ -77,11 +84,11 @@ def explain_team(return context, evaluate_candidates(context)-def _evaluate_and_report(context: CandidateContext) -> list[SuggestionDraft]:+def _evaluate_and_report(context: CandidateContext) -> dict[WarehouseSuggestionKind, CandidateResult]:results = evaluate_candidates(context)for kind, result in results.items():report_candidates(kind, len(result.drafts))- return [draft for result in results.values() for draft in result.drafts]+ return resultsdef is_eligible(reads: TeamReads, rules: EligibilityRules) -> bool:diff --git a/products/warehouse_suggestions/backend/logic/lineage.py b/products/warehouse_suggestions/backend/logic/lineage.pynew file mode 100644index 000000000000..80bbf7d7fccf--- /dev/null+++ b/products/warehouse_suggestions/backend/logic/lineage.py@@ -0,0 +1,20 @@+from collections.abc import Collection, Mapping+from uuid import UUID+++def recomputing_consumers(+ view_id: UUID, dependents: Mapping[UUID, Collection[UUID]], materialized_ids: Collection[UUID]+) -> frozenset[UUID]:+ consumers: set[UUID] = set()+ visited: set[UUID] = {view_id}+ frontier = [view_id]+ while frontier:+ for dependent_id in dependents.get(frontier.pop(), ()):+ if dependent_id in visited:+ continue+ visited.add(dependent_id)+ if dependent_id in materialized_ids:+ consumers.add(dependent_id)+ else:+ frontier.append(dependent_id)+ return frozenset(consumers)diff --git a/products/warehouse_suggestions/backend/logic/reads.py b/products/warehouse_suggestions/backend/logic/reads.pyindex 5f6f53bf4ef5..2e8785ed0991 100644--- a/products/warehouse_suggestions/backend/logic/reads.py+++ b/products/warehouse_suggestions/backend/logic/reads.py@@ -1,6 +1,7 @@from collections import defaultdictfrom collections.abc import Mappingfrom datetime import UTC, date, datetime, time, timedelta+from math import isfinitefrom typing import Anyfrom uuid import UUID@@ -15,6 +16,8 @@ from .rules import Rules, Surface, SurfaceRules, TrafficRulesREADS_QUERY_SETTINGS = {"max_execution_time": 30, "max_threads": 2}FIRST_QUANTILE = 1SECOND_QUANTILE = 2+MEASUREMENT_FEATURE = Feature.WAREHOUSE_SUGGESTIONS+UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)DAYS_FILTER = """day >= %(window_start)sAND day < %(window_end)s"""@@ -77,6 +80,10 @@ class SubjectReads:alone_read_bytes_p90: floatlast_read_at: datetimerequests_by_surface: Mapping[Surface, int]+ measured_reads: int = 0+ measured_duration_ms: float = 0.0+ measured_read_bytes: float = 0.0+ last_measured_at: datetime | None = None@propertydef surfaces(self) -> frozenset[Surface]:@@ -140,6 +147,7 @@ def read_team_reads(team_id: int, window: ReadWindow, rules: Rules, rollup_days:"read": ReadKind.READ.value,"refresh": ReadKind.REFRESH.value,"saved_query": WarehouseSuggestionSubjectKind.SAVED_QUERY.value,+ "measurement_feature": MEASUREMENT_FEATURE.value,}human_sql, human_params = human_condition(rules.traffic)surface_sql, surface_params = surface_expression(rules.surfaces)@@ -227,14 +235,21 @@ SELECTsumIf(read_count, is_human),sumIf(duration_ms_sum, is_human),sumIf(read_bytes_sum, is_human),- sumIf(read_count, read_alone),- quantilesMergeIf(0.5, 0.9)(duration_ms_quantiles, read_alone)[{FIRST_QUANTILE}],- quantilesMergeIf(0.5, 0.9)(read_bytes_quantiles, read_alone)[{FIRST_QUANTILE}],- quantilesMergeIf(0.5, 0.9)(duration_ms_quantiles, read_alone)[{SECOND_QUANTILE}],- quantilesMergeIf(0.5, 0.9)(read_bytes_quantiles, read_alone)[{SECOND_QUANTILE}],- toTimeZone(max(max_event_time), 'UTC')+ sumIf(read_count, read_alone AND NOT is_measurement),+ quantilesMergeIf(0.5, 0.9)(duration_ms_quantiles, read_alone AND NOT is_measurement)[{FIRST_QUANTILE}],+ quantilesMergeIf(0.5, 0.9)(read_bytes_quantiles, read_alone AND NOT is_measurement)[{FIRST_QUANTILE}],+ quantilesMergeIf(0.5, 0.9)(duration_ms_quantiles, read_alone AND NOT is_measurement)[{SECOND_QUANTILE}],+ quantilesMergeIf(0.5, 0.9)(read_bytes_quantiles, read_alone AND NOT is_measurement)[{SECOND_QUANTILE}],+ toTimeZone(max(max_event_time), 'UTC'),+ sumIf(read_count, is_measurement),+ argMaxIf(duration_ms_sum / read_count, max_event_time, is_measurement),+ argMaxIf(read_bytes_sum / read_count, max_event_time, is_measurement),+ toTimeZone(maxIf(max_event_time, is_measurement), 'UTC')FROM (- SELECT *, {{human}} AS is_human+ SELECT+ *,+ {{human}} AS is_human,+ lc_feature = %(measurement_feature)s AND read_alone AS is_measurementFROM {WAREHOUSE_OBJECT_READS_DAILY_TABLE}WHERE {WINDOW_FILTER}AND read_kind = %(read)s@@ -321,10 +336,18 @@ def _subject_reads(alone_read_bytes_p90=float(row[16]),last_read_at=row[17],requests_by_surface=surface_counts.get(subject, {}),+ measured_reads=row[18],+ measured_duration_ms=_finite(row[19]),+ measured_read_bytes=_finite(row[20]),+ last_measured_at=row[21] if row[21] > UNIX_EPOCH else None,)return reads+def _finite(value: float) -> float:+ return float(value) if isfinite(value) else 0.0++def _refreshes(params: Mapping[str, Any], team_id: int) -> dict[UUID, SubjectRefreshes]:refreshes: dict[UUID, SubjectRefreshes] = {}for subject_id, runs, duration_ms, read_bytes in _execute(REFRESHES_SQL, params, team_id):diff --git a/products/warehouse_suggestions/backend/logic/rules.py b/products/warehouse_suggestions/backend/logic/rules.pyindex 80cbd21124b6..a6b15f78415f 100644--- a/products/warehouse_suggestions/backend/logic/rules.py+++ b/products/warehouse_suggestions/backend/logic/rules.py@@ -105,6 +105,14 @@ class MaterializeRules:incremental_lookback_seconds: int+@frozen+class MeasurementRules:+ max_per_team: int+ max_per_run: int+ max_seconds: int+ retry_after_days: int++@frozenclass UnmaterializeRules:min_runs: int@@ -143,6 +151,7 @@ class Rules:certify: CertifyRulesdeprecate: DeprecateRulesmaterialize: MaterializeRules+ measurement: MeasurementRulesunmaterialize: UnmaterializeRulesrefresh: RefreshRuleslifecycle: LifecycleRules@@ -214,6 +223,7 @@ RULES = Rules(min_saved_share=0.5,incremental_lookback_seconds=int(timedelta(days=3).total_seconds()),),+ measurement=MeasurementRules(max_per_team=3, max_per_run=50, max_seconds=180, retry_after_days=7),unmaterialize=UnmaterializeRules(min_runs=3,min_seconds_saved=600.0,diff --git a/products/warehouse_suggestions/backend/logic/view_costs.py b/products/warehouse_suggestions/backend/logic/view_costs.pynew file mode 100644index 000000000000..888a52b4c84e--- /dev/null+++ b/products/warehouse_suggestions/backend/logic/view_costs.py@@ -0,0 +1,103 @@+from datetime import timedelta+from enum import StrEnum+from uuid import UUID++from django.core.cache import cache++from posthog.hogql import ast+from posthog.hogql.constants import HogQLGlobalSettings+from posthog.hogql.query import execute_hogql_query++from posthog.clickhouse.query_tagging import Product, tag_queries+from posthog.clickhouse.workload import Workload+from posthog.dataclasses import frozen+from posthog.errors import ClickHouseQueryTimeOut+from posthog.exceptions_capture import capture_exception+from posthog.models.team import Team++from products.data_modeling.backend.facade.api import get_saved_query_columns, saved_query_definitions++from .candidates.unmaterialize import effective_definition+from .reads import MEASUREMENT_FEATURE+from .rules import MeasurementRules++MEASUREMENT_QUERY_NAME = "warehouse_suggestions_view_cost"+UNFINISHED_KEY_PREFIX = "warehouse_suggestions:unfinished_view_cost"+FAILED_MEASUREMENT = 0+++class MeasurementOutcome(StrEnum):+ MEASURED = "measured"+ CAPPED = "capped"+ FAILED = "failed"+ GONE = "gone"+++@frozen+class UnfinishedMeasurement:+ seconds: int+ fingerprint: str++ @property+ def failed(self) -> bool:+ return self.seconds == FAILED_MEASUREMENT+++def unfinished_measurement(team_id: int, saved_query_id: UUID) -> UnfinishedMeasurement | None:+ remembered = cache.get(_unfinished_key(team_id, saved_query_id))+ if not isinstance(remembered, dict):+ return None+ return UnfinishedMeasurement(seconds=remembered["seconds"], fingerprint=remembered["fingerprint"])+++def measure_view_cost(team_id: int, saved_query_id: UUID, rules: MeasurementRules) -> MeasurementOutcome:+ definitions = saved_query_definitions(team_id, ids=[saved_query_id])+ if not definitions:+ return MeasurementOutcome.GONE+ definition = definitions[0]+ fingerprint = effective_definition(team_id, definition).fingerprint+ column_names = list(get_saved_query_columns(team_id, saved_query_id))+ if not column_names:+ _remember_unfinished(team_id, saved_query_id, rules, seconds=FAILED_MEASUREMENT, fingerprint=fingerprint)+ return MeasurementOutcome.FAILED+ tag_queries(+ product=Product.DATA_MODELING, feature=MEASUREMENT_FEATURE, team_id=team_id, name=MEASUREMENT_QUERY_NAME+ )+ try:+ execute_hogql_query(+ query=full_read_query(definition.name, column_names),+ team=Team.objects.get(id=team_id),+ workload=Workload.OFFLINE,+ settings=HogQLGlobalSettings(max_execution_time=rules.max_seconds),+ bypass_warehouse_access_control=True,+ )+ except ClickHouseQueryTimeOut:+ _remember_unfinished(team_id, saved_query_id, rules, seconds=rules.max_seconds, fingerprint=fingerprint)+ return MeasurementOutcome.CAPPED+ except Exception as error:+ capture_exception(error)+ _remember_unfinished(team_id, saved_query_id, rules, seconds=FAILED_MEASUREMENT, fingerprint=fingerprint)+ return MeasurementOutcome.FAILED+ return MeasurementOutcome.MEASURED+++def full_read_query(view_name: str, column_names: list[str]) -> ast.SelectQuery:+ every_column = ast.Call(name="tuple", args=[ast.Field(chain=[name]) for name in column_names])+ row_hash = ast.Call(name="cityHash64", args=[ast.Call(name="toString", args=[every_column])])+ return ast.SelectQuery(+ select=[ast.Call(name="count", args=[]), ast.Call(name="sum", args=[row_hash])],+ select_from=ast.JoinExpr(table=ast.Field(chain=[view_name])),+ )+++def _remember_unfinished(+ team_id: int, saved_query_id: UUID, rules: MeasurementRules, *, seconds: int, fingerprint: str+) -> None:+ timeout = int(timedelta(days=rules.retry_after_days).total_seconds())+ cache.set(+ _unfinished_key(team_id, saved_query_id), {"seconds": seconds, "fingerprint": fingerprint}, timeout=timeout+ )+++def _unfinished_key(team_id: int, saved_query_id: UUID) -> str:+ return f"{UNFINISHED_KEY_PREFIX}:{team_id}:{saved_query_id}"diff --git a/products/warehouse_suggestions/backend/presentation/serializers.py b/products/warehouse_suggestions/backend/presentation/serializers.pyindex 7566d3383f8a..3a939f28945c 100644--- a/products/warehouse_suggestions/backend/presentation/serializers.py+++ b/products/warehouse_suggestions/backend/presentation/serializers.py@@ -97,6 +97,7 @@ class MaterializePayloadSerializer(DataclassSerializer):allow_null=True,help_text="Why the interval is not shorter. source_sync: the view's sources sync no faster. ""fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. "+ "consumer_refresh: the materialized views that read it refresh no more often. ""savings: a shorter interval would cost more than it saves. Null on suggestions made before this was ""recorded.",)@@ -125,6 +126,10 @@ class MaterializePayloadSerializer(DataclassSerializer):"help_text": "How far before the last refresh's highest incremental_key value each refresh reads ""again, in seconds. The last incremental_key value is always read again."},+ "recomputed_by_views": {+ "help_text": "How many materialized views compute this view again on every refresh, because it "+ "is not stored. 0 when the saving comes only from people's reads."+ },}diff --git a/products/warehouse_suggestions/backend/temporal/__init__.py b/products/warehouse_suggestions/backend/temporal/__init__.pyindex 45b1a20aaa21..849a51226f4d 100644--- a/products/warehouse_suggestions/backend/temporal/__init__.py+++ b/products/warehouse_suggestions/backend/temporal/__init__.py@@ -5,11 +5,15 @@ from typing import Anyfrom posthog.temporal.common.base import PostHogWorkflow-from .activities import generate_warehouse_suggestions, get_warehouse_suggestion_team_batches+from .activities import generate_warehouse_suggestions, get_warehouse_suggestion_team_batches, measure_view_costsfrom .workflows import WarehouseSuggestionsWorkflowWORKFLOWS: list[type[PostHogWorkflow]] = [WarehouseSuggestionsWorkflow]-ACTIVITIES: list[Callable[..., Any]] = [get_warehouse_suggestion_team_batches, generate_warehouse_suggestions]+ACTIVITIES: list[Callable[..., Any]] = [+ get_warehouse_suggestion_team_batches,+ generate_warehouse_suggestions,+ measure_view_costs,+]__all__ = ["ACTIVITIES", "WORKFLOWS"]diff --git a/products/warehouse_suggestions/backend/temporal/activities.py b/products/warehouse_suggestions/backend/temporal/activities.pyindex e3e715e38b55..1593586631c2 100644--- a/products/warehouse_suggestions/backend/temporal/activities.py+++ b/products/warehouse_suggestions/backend/temporal/activities.py@@ -1,5 +1,6 @@from collections import Counterfrom itertools import batched+from uuid import UUIDfrom django.utils import timezone@@ -19,7 +20,8 @@ from ..logic.flags import is_warehouse_suggestions_enabledfrom ..logic.job import TeamRunStatus, run_team, team_ids_with_suggestions_to_maintainfrom ..logic.reads import ReadWindow, read_rollup_daysfrom ..logic.rules import RULES-from .contracts import BatchOutcome, WarehouseSuggestionsInputs+from ..logic.view_costs import MeasurementOutcome, measure_view_cost+from .contracts import BatchOutcome, ViewCostRequest, WarehouseSuggestionsInputsLOGGER = get_logger(__name__)FULL_ROLLOUT = 1.0@@ -53,18 +55,35 @@ def run_batch(team_ids: list[int], run_id: str) -> BatchOutcome:today = timezone.now().date()rollup_days = read_rollup_days(ReadWindow.ending(today, RULES))outcomes: Counter[str] = Counter()+ view_costs_to_measure: list[ViewCostRequest] = []for team_id in team_ids:try:- outcomes[run_team(team_id, run_id=run_id, today=today, rollup_days=rollup_days).status] += 1+ result = run_team(team_id, run_id=run_id, today=today, rollup_days=rollup_days)except Exception:LOGGER.exception("warehouse_suggestions.team_failed", team_id=team_id)outcomes["failed"] += 1+ continue+ outcomes[result.status] += 1+ view_costs_to_measure.extend(+ ViewCostRequest(team_id=team_id, saved_query_id=str(subject.id)) for subject in result.measurement_requests+ )return BatchOutcome(processed=outcomes[TeamRunStatus.PROCESSED],not_eligible=outcomes[TeamRunStatus.NOT_ELIGIBLE],disabled=outcomes[TeamRunStatus.DISABLED],failed=outcomes["failed"],+ view_costs_to_measure=view_costs_to_measure[: RULES.measurement.max_per_run],+ )+++def measure_view_costs_sync(requests: list[ViewCostRequest]) -> int:+ outcomes: Counter[MeasurementOutcome] = Counter()+ for request in requests:+ outcomes[measure_view_cost(request.team_id, UUID(request.saved_query_id), RULES.measurement)] += 1+ LOGGER.info(+ "warehouse_suggestions.view_costs_measured", **{outcome.value: count for outcome, count in outcomes.items()})+ return outcomes.total()def _teams_to_run() -> set[int]:@@ -87,3 +106,9 @@ async def get_warehouse_suggestion_team_batches(inputs: WarehouseSuggestionsInpuasync def generate_warehouse_suggestions(team_ids: list[int], run_id: str) -> BatchOutcome:async with Heartbeater():return await database_sync_to_async_pool(run_batch)(team_ids, run_id)+++async def measure_view_costs(requests: list[ViewCostRequest]) -> int:+ async with Heartbeater():+ return await database_sync_to_async_pool(measure_view_costs_sync)(requests)diff --git a/products/warehouse_suggestions/backend/temporal/contracts.py b/products/warehouse_suggestions/backend/temporal/contracts.pyindex c6c55112120e..4c4d36924e0a 100644--- a/products/warehouse_suggestions/backend/temporal/contracts.py+++ b/products/warehouse_suggestions/backend/temporal/contracts.py@@ -1,4 +1,4 @@-from dataclasses import dataclass+from dataclasses import dataclass, fieldWAREHOUSE_SUGGESTIONS_WORKFLOW_NAME = "warehouse-suggestions"@@ -11,9 +11,16 @@ class WarehouseSuggestionsInputs:team_ids: list[int] | None = None+@dataclass(frozen=True, kw_only=True)+class ViewCostRequest:+ team_id: int+ saved_query_id: str++@dataclass(frozen=True, kw_only=True)class BatchOutcome:processed: int = 0not_eligible: int = 0disabled: int = 0failed: int = 0+ view_costs_to_measure: list[ViewCostRequest] = field(default_factory=list)diff --git a/products/warehouse_suggestions/backend/temporal/workflows.py b/products/warehouse_suggestions/backend/temporal/workflows.pyindex bdecd6cf4657..8a8ce6f14b72 100644--- a/products/warehouse_suggestions/backend/temporal/workflows.py+++ b/products/warehouse_suggestions/backend/temporal/workflows.py@@ -7,9 +7,15 @@ from temporalio.common import RetryPolicyfrom posthog.temporal.common.base import PostHogWorkflow-from .activities import generate_warehouse_suggestions, get_warehouse_suggestion_team_batches+from ..logic.rules import RULES+from .activities import generate_warehouse_suggestions, get_warehouse_suggestion_team_batches, measure_view_costsfrom .contracts import WAREHOUSE_SUGGESTIONS_WORKFLOW_NAME, BatchOutcome, WarehouseSuggestionsInputs+MEASURE_VIEW_COSTS_PATCH = "measure-view-costs"+MEASUREMENT_MARGIN = dt.timedelta(minutes=10)+MEASUREMENT_TIMEOUT = (+ dt.timedelta(seconds=RULES.measurement.max_per_run * RULES.measurement.max_seconds) + MEASUREMENT_MARGIN+)RETRY_POLICY = RetryPolicy(maximum_attempts=3,initial_interval=dt.timedelta(seconds=30),@@ -55,4 +61,15 @@ class WarehouseSuggestionsWorkflow(PostHogWorkflow):for field in dataclasses.fields(BatchOutcome)})+ total = dataclasses.replace(+ total, view_costs_to_measure=total.view_costs_to_measure[: RULES.measurement.max_per_run]+ )+ if total.view_costs_to_measure and workflow.patched(MEASURE_VIEW_COSTS_PATCH):+ await workflow.execute_activity(+ measure_view_costs,+ total.view_costs_to_measure,+ start_to_close_timeout=MEASUREMENT_TIMEOUT,+ heartbeat_timeout=dt.timedelta(minutes=5),+ retry_policy=RetryPolicy(maximum_attempts=1),+ )return totaldiff --git a/products/warehouse_suggestions/backend/tests/test_candidates.py b/products/warehouse_suggestions/backend/tests/test_candidates.pyindex 85b1d231ceed..9c55771f9907 100644--- a/products/warehouse_suggestions/backend/tests/test_candidates.py+++ b/products/warehouse_suggestions/backend/tests/test_candidates.py@@ -1,5 +1,6 @@from dataclasses import replacefrom datetime import timedelta+from types import SimpleNamespacefrom typing import Anyfrom uuid import uuid4@@ -40,9 +41,13 @@ from products.warehouse_suggestions.backend.logic.candidates.materialize importinterval_basis,refresh_cost,)-from products.warehouse_suggestions.backend.logic.candidates.unmaterialize import UnmaterializeCandidate+from products.warehouse_suggestions.backend.logic.candidates.unmaterialize import (+ EffectiveDefinition,+ UnmaterializeCandidate,+)from products.warehouse_suggestions.backend.logic.reads import RefreshRun, SubjectRefreshesfrom products.warehouse_suggestions.backend.logic.rules import RULES, Surface+from products.warehouse_suggestions.backend.logic.view_costs import UnfinishedMeasurementfrom products.warehouse_suggestions.backend.tests.factories import (busy_reads,context,@@ -571,3 +576,116 @@ class TestMaterializeAfterUnmaterialize(SimpleTestCase):(),["was unmaterialized on a suggestion in the last 30 days"],)+++MATERIALIZE_MODULE = "products.warehouse_suggestions.backend.logic.candidates.materialize"+FIRST_CONSUMER, SECOND_CONSUMER = uuid4(), uuid4()+CURRENT_DEFINITION, EARLIER_DEFINITION = "current", "earlier"+TIME_LIMIT = RULES.measurement.max_seconds+EVERY_BUCKET_ALLOWED = SimpleNamespace(+ bounds=SimpleNamespace(allowed=(timedelta(minutes=15), HOUR, timedelta(hours=6), DAY), floor=None),+ max_data_age=timedelta(0),+ best_effort_source_ids=set(),+ identities={},+)+UNREAD = {+ "human_requests": 0,+ "human_reads": 0,+ "human_days": 0,+ "human_duration_ms": 0,+ "human_read_bytes": 0,+ "alone_reads": 0,+}+CHEAP_PREVIEWS = {+ "human_requests": HOURLY_RUNS,+ "human_reads": HOURLY_RUNS,+ "human_duration_ms": HOURLY_RUNS * 500,+ "human_read_bytes": HOURLY_RUNS * 10**6,+ "alone_duration_ms_median": 500.0,+ "alone_read_bytes_median": float(10**6),+}+++class TestMaterializeRecomputedUpstream(SimpleTestCase):+ @parameterized.expand(+ [+ ("one_consumer_refreshes_it_as_often_as_a_stored_copy_would", 1, 0, UNREAD, 3, None, None, False, False),+ ("two_consumers_with_a_measured_cost", 2, 0, UNREAD, 3, None, None, True, False),+ ("two_consumers_and_no_measurement", 2, 0, UNREAD, None, None, None, False, True),+ ("measured_before_the_last_edit", 2, 0, UNREAD, 3, 1, None, False, True),+ ("measurement_hit_the_time_limit", 2, 0, UNREAD, None, None, (TIME_LIMIT, CURRENT_DEFINITION), True, False),+ ("time_limit_hit_before_an_edit", 2, 0, UNREAD, None, None, (TIME_LIMIT, EARLIER_DEFINITION), False, True),+ ("measurement_failed", 2, 0, UNREAD, None, None, (0, CURRENT_DEFINITION), False, False),+ ("measurement_failed_before_an_edit", 2, 0, UNREAD, None, None, (0, EARLIER_DEFINITION), False, True),+ ("an_incremental_consumer_does_not_count", 2, 1, UNREAD, 3, None, None, False, False),+ ("one_consumer_and_cheap_previews", 1, 0, CHEAP_PREVIEWS, 3, None, None, False, False),+ ]+ )+ def test_proposes_a_view_that_materialized_views_compute_on_every_refresh(+ self,+ _name: str,+ consumers: int,+ incremental_consumers: int,+ read_overrides: dict,+ measured_days_ago: int | None,+ edited_days_ago: int | None,+ unfinished: tuple[int, str] | None,+ expect_draft: bool,+ expect_measurement_request: bool,+ ) -> None:+ consumer_ids = [FIRST_CONSUMER, SECOND_CONSUMER][:consumers]+ reads = busy_reads(+ **read_overrides,+ measured_reads=1 if measured_days_ago else 0,+ measured_duration_ms=TWENTY_SECONDS_MS,+ last_measured_at=days_ago(measured_days_ago) if measured_days_ago else None,+ )+ candidate_context = context(+ team_reads(+ {view_subject(VIEW_ID): reads},+ refreshes={+ consumer_id: SubjectRefreshes(runs=HOURLY_RUNS, duration_ms=HOURLY_RUNS * 60_000, read_bytes=1)+ for consumer_id in consumer_ids+ },+ ),+ views=[+ view(VIEW_ID),+ *(+ view(consumer_id, materializes=True, refreshes_incrementally=index < incremental_consumers)+ for index, consumer_id in enumerate(consumer_ids)+ ),+ ],+ )++ with (+ patch(f"{MATERIALIZE_MODULE}.dependent_saved_query_ids", return_value={VIEW_ID: frozenset(consumer_ids)}),+ patch(f"{MATERIALIZE_MODULE}.saved_query_target_bounds", return_value=EVERY_BUCKET_ALLOWED),+ patch(+ f"{MATERIALIZE_MODULE}.effective_definition",+ return_value=EffectiveDefinition(+ fingerprint=CURRENT_DEFINITION,+ changed_at=days_ago(edited_days_ago) if edited_days_ago else None,+ ),+ ),+ patch(+ f"{MATERIALIZE_MODULE}.unfinished_measurement",+ return_value=UnfinishedMeasurement(seconds=unfinished[0], fingerprint=unfinished[1])+ if unfinished+ else None,+ ),+ patch(f"{MATERIALIZE_MODULE}.upstream_table_refs", return_value=[]),+ patch(f"{MATERIALIZE_MODULE}.definition_database", return_value=None),+ ):+ result = MaterializeCandidate().evaluate(candidate_context)++ assert result.measurement_requests == ((view_subject(VIEW_ID),) if expect_measurement_request else ())+ if not expect_draft:+ assert result.drafts == ()+ return+ [draft] = result.drafts+ assert isinstance(draft.payload, MaterializePayload)+ assert (+ draft.payload.refresh_interval_seconds,+ draft.payload.recomputed_by_views,+ draft.payload.refresh_interval_basis,+ ) == (HOUR.total_seconds(), 2, WarehouseSuggestionRefreshIntervalBasis.CONSUMER_REFRESH)diff --git a/products/warehouse_suggestions/backend/tests/test_lineage.py b/products/warehouse_suggestions/backend/tests/test_lineage.pynew file mode 100644index 000000000000..91561032a1a6--- /dev/null+++ b/products/warehouse_suggestions/backend/tests/test_lineage.py@@ -0,0 +1,28 @@+from uuid import uuid4++from django.test import SimpleTestCase++from parameterized import parameterized++from products.warehouse_suggestions.backend.logic.lineage import recomputing_consumers++VIEW, MIDDLE, CONSUMER, BEYOND = uuid4(), uuid4(), uuid4(), uuid4()+++class TestRecomputingConsumers(SimpleTestCase):+ @parameterized.expand(+ [+ ("through_a_view_that_is_not_materialized", {VIEW: [MIDDLE], MIDDLE: [CONSUMER]}, {CONSUMER}, {CONSUMER}),+ (+ "stops_at_the_first_materialized_view",+ {VIEW: [CONSUMER], CONSUMER: [BEYOND]},+ {CONSUMER, BEYOND},+ {CONSUMER},+ ),+ ("survives_a_cycle", {VIEW: [MIDDLE], MIDDLE: [VIEW, CONSUMER]}, {CONSUMER}, {CONSUMER}),+ ]+ )+ def test_finds_the_materialized_views_whose_refreshes_compute_it(+ self, _name: str, dependents: dict, materialized: set, expected: set+ ) -> None:+ assert recomputing_consumers(VIEW, dependents, materialized) == expecteddiff --git a/products/warehouse_suggestions/backend/tests/test_reads.py b/products/warehouse_suggestions/backend/tests/test_reads.pyindex 51a855a07ae8..07f692feae9a 100644--- a/products/warehouse_suggestions/backend/tests/test_reads.py+++ b/products/warehouse_suggestions/backend/tests/test_reads.py@@ -84,6 +84,32 @@ class TestReadTeamReads(ClickhouseTestMixin, BaseTest):RollupRead(subject_id=view, day=TODAY - timedelta(days=40), request_id="old", user_id=3),RollupRead(subject_id=view, day=TODAY, request_id="today", user_id=4),RollupRead(subject_id=view, day=YESTERDAY, request_id="warmup", user_id=0, duration_ms=50),+ RollupRead(+ subject_id=view,+ day=YESTERDAY,+ request_id="consumer-refresh",+ user_id=0,+ lc_kind="temporal",+ lc_feature="data_modeling",+ duration_ms=4000,+ ),+ *(+ RollupRead(+ subject_id=view,+ day=day,+ request_id=f"measurement-{duration_ms}",+ user_id=0,+ lc_kind="temporal",+ lc_feature="warehouse_suggestions",+ read_alone=True,+ duration_ms=duration_ms,+ read_bytes=read_bytes,+ )+ for day, duration_ms, read_bytes in (+ (TODAY - timedelta(days=2), 70_000, 50_000),+ (YESTERDAY, 2000, 1000),+ )+ ),RollupRead(subject_id=view,day=YESTERDAY,@@ -136,7 +162,9 @@ class TestReadTeamReads(ClickhouseTestMixin, BaseTest):assert rollup_days == RollupDays(days_with_data=3, recent_days_with_data=3)assert (reads.readers, reads.view_readers, reads.view_reads) == (3, 2, 2)assert (view_reads.human_requests, view_reads.human_users, view_reads.human_days) == (2, 2, 2)- assert (view_reads.background_reads, view_reads.background_duration_ms) == (1, 50)+ assert (view_reads.background_reads, view_reads.background_duration_ms) == (4, 76050)+ assert (view_reads.measured_duration_ms, view_reads.measured_read_bytes) == (2000, 1000)+ assert view_reads.last_measured_at == datetime.combine(YESTERDAY, time(hour=12), tzinfo=UTC)refreshes = reads.refreshes[view]assert (refreshes.runs, refreshes.duration_ms) == (2, 4090)assert {run.workflow_id: run.duration_ms for run in reads.refresh_runs[view]} == {"run-1": 1000, "run-2": 3090}diff --git a/products/warehouse_suggestions/backend/tests/test_temporal.py b/products/warehouse_suggestions/backend/tests/test_temporal.pyindex fd7fb9ea9a74..ad5c88ed57c7 100644--- a/products/warehouse_suggestions/backend/tests/test_temporal.py+++ b/products/warehouse_suggestions/backend/tests/test_temporal.py@@ -15,7 +15,9 @@ from products.warehouse_suggestions.backend.facade.enums import WarehouseSuggestfrom products.warehouse_suggestions.backend.logic.candidates.registry import CANDIDATESfrom products.warehouse_suggestions.backend.logic.job import TeamRunResult, TeamRunStatusfrom products.warehouse_suggestions.backend.logic.reads import RollupDays+from products.warehouse_suggestions.backend.logic.rules import RULESfrom products.warehouse_suggestions.backend.logic.suggestions import upsert_suggestions+from products.warehouse_suggestions.backend.logic.view_costs import MeasurementOutcomefrom products.warehouse_suggestions.backend.models import WarehouseSuggestionfrom products.warehouse_suggestions.backend.temporal import (ACTIVITIES as WAREHOUSE_SUGGESTIONS_ACTIVITIES,@@ -87,15 +89,22 @@ class TestTeamBatches(BaseTest):assert batches == [[self.team.pk]]-async def test_a_worker_runs_a_nonempty_batch_without_failing_a_team() -> None:+async def test_a_worker_runs_a_nonempty_batch_and_measures_the_views_a_team_asked_for() -> None:task_queue = str(uuid.uuid4())+ unmeasured_view = uuid.uuid4()+ measurement_requests = {1: (view_subject(unmeasured_view),), 2: ()}with (patch(f"{ACTIVITIES}.team_batches", return_value=[[1, 2]]),patch(f"{ACTIVITIES}.read_rollup_days", return_value=RollupDays(days_with_data=30, recent_days_with_data=7)),patch(f"{ACTIVITIES}.run_team",- side_effect=lambda team_id, **_: TeamRunResult(team_id=team_id, status=TeamRunStatus.PROCESSED),+ side_effect=lambda team_id, **_: TeamRunResult(+ team_id=team_id,+ status=TeamRunStatus.PROCESSED,+ measurement_requests=measurement_requests[team_id],+ ),),+ patch(f"{ACTIVITIES}.measure_view_cost", return_value=MeasurementOutcome.MEASURED) as measure_view_cost,):async with await WorkflowEnvironment.start_time_skipping() as environment:async with Worker(@@ -113,4 +122,5 @@ async def test_a_worker_runs_a_nonempty_batch_without_failing_a_team() -> None:result_type=BatchOutcome,)- assert outcome == BatchOutcome(processed=2)+ assert outcome.processed == 2+ measure_view_cost.assert_called_once_with(1, unmeasured_view, RULES.measurement)diff --git a/products/warehouse_suggestions/backend/tests/test_view_costs.py b/products/warehouse_suggestions/backend/tests/test_view_costs.pynew file mode 100644index 000000000000..7aba395ba2bc--- /dev/null+++ b/products/warehouse_suggestions/backend/tests/test_view_costs.py@@ -0,0 +1,66 @@+from posthog.test.base import BaseTest, ClickhouseTestMixin+from unittest.mock import patch++from django.core.cache import cache++from parameterized import parameterized++from posthog.errors import ClickHouseQueryTimeOut++from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery+from products.warehouse_suggestions.backend.logic.rules import RULES+from products.warehouse_suggestions.backend.logic.view_costs import (+ FAILED_MEASUREMENT,+ MeasurementOutcome,+ measure_view_cost,+ unfinished_measurement,+)++VIEW_COSTS_MODULE = "products.warehouse_suggestions.backend.logic.view_costs"+++class TestMeasureViewCost(ClickhouseTestMixin, BaseTest):+ def setUp(self) -> None:+ super().setUp()+ cache.clear()+ self.saved_query = DataWarehouseSavedQuery.objects.create(+ team=self.team,+ name="daily_events",+ query={"kind": "HogQLQuery", "query": "SELECT toDate(timestamp) AS day, event FROM events"},+ columns={+ "day": {"hogql": "DateDatabaseField", "clickhouse": "Date"},+ "event": {"hogql": "StringDatabaseField", "clickhouse": "String"},+ },+ )++ def test_reads_every_column_of_the_view(self) -> None:+ outcome = measure_view_cost(self.team.pk, self.saved_query.id, RULES.measurement)++ assert (outcome, unfinished_measurement(self.team.pk, self.saved_query.id)) == (+ MeasurementOutcome.MEASURED,+ None,+ )++ @parameterized.expand(+ [+ ("timeout", ClickHouseQueryTimeOut(), True, MeasurementOutcome.CAPPED, RULES.measurement.max_seconds),+ ("other_error", ValueError("broken view"), True, MeasurementOutcome.FAILED, FAILED_MEASUREMENT),+ ("no_stored_columns", None, False, MeasurementOutcome.FAILED, FAILED_MEASUREMENT),+ ]+ )+ def test_a_measurement_that_does_not_finish_is_remembered(+ self,+ _name: str,+ error: Exception | None,+ has_columns: bool,+ expected_outcome: MeasurementOutcome,+ expected_cap: int,+ ) -> None:+ if not has_columns:+ DataWarehouseSavedQuery.objects.filter(id=self.saved_query.id).update(columns={})++ with patch(f"{VIEW_COSTS_MODULE}.execute_hogql_query", side_effect=error):+ outcome = measure_view_cost(self.team.pk, self.saved_query.id, RULES.measurement)++ unfinished = unfinished_measurement(self.team.pk, self.saved_query.id)+ assert (outcome, unfinished.seconds if unfinished else None) == (expected_outcome, expected_cap)diff --git a/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx b/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsxindex 39c1ed8961b4..56362e845eef 100644--- a/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx+++ b/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx@@ -6,7 +6,13 @@ import { LemonButton } from '@posthog/lemon-ui'import { iconForType } from '~/layout/panel-layout/ProjectTree/defaultTree'import type { WarehouseSuggestionApi } from '../generated/api.schemas'-import { isMaterializePayload, materializeSaving, readSummary, savingHighlight } from '../suggestionCopy'+import {+ isMaterializePayload,+ materializeSaving,+ readSummary,+ rebuiltSummary,+ savingHighlight,+} from '../suggestionCopy'import { warehouseSuggestionsLogic } from '../warehouseSuggestionsLogic'import { DismissSuggestionButtons } from './DismissSuggestionButtons'import { SuggestionRow } from './SuggestionRow'@@ -31,6 +37,8 @@ export function MaterializeSuggestionRow({return null}const leaving = leavingSuggestionIds.includes(suggestion.id)+ const rebuiltByViews = payload.recomputed_by_views ?? 0+ const readByPeople = (suggestion.evidence.human_requests ?? 0) > 0const blockedReason = !suggestion.can_act? EDIT_ACCESS_REASON: actionsInFlight[suggestion.id] || leaving@@ -46,13 +54,16 @@ export function MaterializeSuggestionRow({icon={iconForType('data_modeling')}title={<SuggestionSubjectLink suggestion={suggestion} />}meta={[- <WhyThisSuggestion- key="reads"- evidence={suggestion.evidence}- windowDays={windowDays}- label={readSummary(suggestion.evidence)}- />,- ]}+ rebuiltByViews > 0 && <span key="rebuilt">{rebuiltSummary(rebuiltByViews)}</span>,+ (readByPeople || rebuiltByViews === 0) && (+ <WhyThisSuggestion+ key="reads"+ evidence={suggestion.evidence}+ windowDays={windowDays}+ label={readSummary(suggestion.evidence)}+ />+ ),+ ].filter(Boolean)}highlight={savingHighlight(materializeSaving(payload))}actions={<>diff --git a/products/warehouse_suggestions/frontend/generated/api.schemas.ts b/products/warehouse_suggestions/frontend/generated/api.schemas.tsindex e13e3bb7a673..14b4bc852c9d 100644--- a/products/warehouse_suggestions/frontend/generated/api.schemas.ts+++ b/products/warehouse_suggestions/frontend/generated/api.schemas.ts@@ -44,6 +44,7 @@ export const WarehouseSuggestionRefreshModeEnumApi = {* * `source_sync` - Source sync* * `fastest_schedule` - Fastest schedule* * `read_rate` - Read rate+ * * `consumer_refresh` - Consumer refresh* * `savings` - Savings*/export type WarehouseSuggestionRefreshIntervalBasisEnumApi =@@ -53,6 +54,7 @@ export const WarehouseSuggestionRefreshIntervalBasisEnumApi = {SourceSync: 'source_sync',FastestSchedule: 'fastest_schedule',ReadRate: 'read_rate',+ ConsumerRefresh: 'consumer_refresh',Savings: 'savings',} as const@@ -66,11 +68,12 @@ export interface WarehouseSuggestionMaterializePayloadApi {* * `full` - Full* * `incremental` - Incremental */refresh_mode: WarehouseSuggestionRefreshModeEnumApi- /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.+ /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. consumer_refresh: the materialized views that read it refresh no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.** * `source_sync` - Source sync* * `fastest_schedule` - Fastest schedule* * `read_rate` - Read rate+ * * `consumer_refresh` - Consumer refresh* * `savings` - Savings */refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasisEnumApi | null/** Name of the view to materialize. */@@ -97,6 +100,8 @@ export interface WarehouseSuggestionMaterializePayloadApi {unique_key?: string[]/** How far before the last refresh's highest incremental_key value each refresh reads again, in seconds. The last incremental_key value is always read again. */lookback_seconds?: number+ /** How many materialized views compute this view again on every refresh, because it is not stored. 0 when the saving comes only from people's reads. */+ recomputed_by_views?: number}export interface WarehouseSuggestionUnmaterializePayloadApi {diff --git a/products/warehouse_suggestions/frontend/suggestionCopy.test.ts b/products/warehouse_suggestions/frontend/suggestionCopy.test.tsindex 1a3178da66a4..2a648f852dd7 100644--- a/products/warehouse_suggestions/frontend/suggestionCopy.test.ts+++ b/products/warehouse_suggestions/frontend/suggestionCopy.test.ts@@ -81,6 +81,7 @@ describe('suggestionCopy', () => {it.each<[Parameters<typeof suggestedScheduleHint>[0], string]>([['source_sync', "sources don't sync more often"],['read_rate', "People don't read it more often"],+ ['consumer_refresh', "The views that read it don't refresh more often"],['fastest_schedule', 'the most often PostHog refreshes a view'],['savings', 'saves more than it costs'],[null, 'saves more than it costs'],diff --git a/products/warehouse_suggestions/frontend/suggestionCopy.ts b/products/warehouse_suggestions/frontend/suggestionCopy.tsindex 1e4b31c2be8f..0f84b7bc2dc5 100644--- a/products/warehouse_suggestions/frontend/suggestionCopy.ts+++ b/products/warehouse_suggestions/frontend/suggestionCopy.ts@@ -119,6 +119,8 @@ export function suggestedScheduleHint(return `Suggested: every ${cadence}. Its sources don't sync more often than that.`case 'read_rate':return `Suggested: every ${cadence}. People don't read it more often than that.`+ case 'consumer_refresh':+ return `Suggested: every ${cadence}. The views that read it don't refresh more often than that.`case 'fastest_schedule':return `Suggested: every ${cadence}, the most often PostHog refreshes a view.`default:@@ -135,6 +137,8 @@ export function shorterScheduleWarning(return `Its sources sync every ${cadence} at most, so refreshing more often reads the same data again.`case 'read_rate':return `People don't read it more often than every ${cadence}, so extra refreshes mostly go unread.`+ case 'consumer_refresh':+ return `The views that read it refresh every ${cadence} at most, so refreshing more often goes unread.`case 'fastest_schedule':return nulldefault:@@ -207,6 +211,10 @@ export function readSummary(evidence: WarehouseSuggestionEvidenceApi): string {return `${countOf(evidence.human_requests ?? 0, 'read', 'reads')} by ${countOf(evidence.human_users ?? 0, 'person', 'people')}`}+export function rebuiltSummary(views: number): string {+ return `Rebuilt on every refresh of ${countOf(views, 'view', 'views')}`+}+export function daysReadLabel(evidence: WarehouseSuggestionEvidenceApi, windowDays: number): string {return `Read on ${evidence.human_days ?? 0} of the last ${windowDays} days`}diff --git a/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md b/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.mdindex 32abda5cea97..7a3a503e484d 100644--- a/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md+++ b/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md@@ -29,11 +29,12 @@ Its `evidence` has the read counts behind it.Tell the user what accepting does and what it costs, using the payload fields.Do not invent a score or a confidence level.-- **materialize**: the view is read often enough that storing its result saves compute.- - `saves_seconds_per_month` and `saves_bytes_per_month` are what the reads stop costing.+- **materialize**: the view is computed often enough that storing its result saves compute.+ - A view that is not stored is computed on every read, and again on every refresh of each materialized view that reads it. `recomputed_by_views` counts those materialized views. When it is above 0, the saving comes mostly from their refreshes, not from people's reads.+ - `saves_seconds_per_month` and `saves_bytes_per_month` are what those computations stop costing.- Materializing adds a recurring refresh cost, which these numbers already subtract.- `freshness_today_seconds` and `freshness_after_seconds` say how old the data can be now and after.- - `refresh_interval_seconds` is the proposed interval. `refresh_interval_basis` says what kept it from being shorter: `source_sync` (the sources sync no faster), `fastest_schedule` (PostHog refreshes views no more often), `read_rate` (people read the view no more often) or `savings` (a shorter interval costs more than it saves).+ - `refresh_interval_seconds` is the proposed interval. `refresh_interval_basis` says what kept it from being shorter: `source_sync` (the sources sync no faster), `fastest_schedule` (PostHog refreshes views no more often), `read_rate` (people read the view no more often), `consumer_refresh` (the materialized views that read it refresh no more often) or `savings` (a shorter interval costs more than it saves).- `refresh_mode` is `incremental` when each refresh can read only new rows, by `incremental_key`, and update rows by `unique_key`. Otherwise it is `full`, and each refresh rebuilds the whole table.- **unmaterialize**: this materialized view is read so rarely that refreshing it costs more than computing it on each read.- `refresh_seconds_per_month` and `refresh_bytes_per_month` are what its refreshes cost now.diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex d920f39d2b5f..a2b6ef01eec1 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -83020,6 +83020,7 @@ export namespace Schemas {* * `source_sync` - Source sync* * `fastest_schedule` - Fastest schedule* * `read_rate` - Read rate+ * * `consumer_refresh` - Consumer refresh* * `savings` - Savings*/export type WarehouseSuggestionRefreshIntervalBasisEnum = typeof WarehouseSuggestionRefreshIntervalBasisEnum[keyof typeof WarehouseSuggestionRefreshIntervalBasisEnum];@@ -83029,6 +83030,7 @@ export namespace Schemas {SourceSync: 'source_sync',FastestSchedule: 'fastest_schedule',ReadRate: 'read_rate',+ ConsumerRefresh: 'consumer_refresh',Savings: 'savings',} as const;@@ -83042,11 +83044,12 @@ export namespace Schemas {* * `full` - Full* * `incremental` - Incremental */refresh_mode: WarehouseSuggestionRefreshModeEnum;- /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.+ /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. consumer_refresh: the materialized views that read it refresh no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.** * `source_sync` - Source sync* * `fastest_schedule` - Fastest schedule* * `read_rate` - Read rate+ * * `consumer_refresh` - Consumer refresh* * `savings` - Savings */refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasisEnum | null;/** Name of the view to materialize. */@@ -83073,6 +83076,8 @@ export namespace Schemas {unique_key?: string[];/** How far before the last refresh's highest incremental_key value each refresh reads again, in seconds. The last incremental_key value is always read again. */lookback_seconds?: number;+ /** How many materialized views compute this view again on every refresh, because it is not stored. 0 when the saving comes only from people's reads. */+ recomputed_by_views?: number;}export interface WarehouseSuggestionUnmaterializePayload {