Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-suggestions): suggest materializing views that refreshes recompute (#124125)

Thiago Salvatore7271fe706696
diff --git a/products/warehouse_suggestions/backend/facade/contracts.py b/products/warehouse_suggestions/backend/facade/contracts.py
index b8c5459de2d6..08c6a09b002c 100644
--- a/products/warehouse_suggestions/backend/facade/contracts.py
+++ b/products/warehouse_suggestions/backend/facade/contracts.py
@@ -60,6 +60,7 @@ class MaterializePayload:
unique_key: tuple[str, ...] = ()
lookback_seconds: int = 0
refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasis | None = None
+ recomputed_by_views: int = 0
@frozen
@@ -99,6 +100,7 @@ class MaterializeSuggestionPayload:
unique_key: tuple[str, ...] = ()
lookback_seconds: int = 0
refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasis | None = None
+ recomputed_by_views: int = 0
SuggestionPayloadView = CertifyPayload | DeprecatePayload | MaterializeSuggestionPayload | UnmaterializePayload
diff --git a/products/warehouse_suggestions/backend/facade/enums.py b/products/warehouse_suggestions/backend/facade/enums.py
index 25a8ba1a8002..ad05c93ac071 100644
--- a/products/warehouse_suggestions/backend/facade/enums.py
+++ b/products/warehouse_suggestions/backend/facade/enums.py
@@ -44,4 +44,5 @@ class WarehouseSuggestionRefreshIntervalBasis(LabeledStrEnum):
SOURCE_SYNC = ("source_sync", "Source sync")
FASTEST_SCHEDULE = ("fastest_schedule", "Fastest schedule")
READ_RATE = ("read_rate", "Read rate")
+ CONSUMER_REFRESH = ("consumer_refresh", "Consumer refresh")
SAVINGS = "savings"
diff --git a/products/warehouse_suggestions/backend/logic/candidates/base.py b/products/warehouse_suggestions/backend/logic/candidates/base.py
index 812c28f50f0d..cbd4580f90f3 100644
--- a/products/warehouse_suggestions/backend/logic/candidates/base.py
+++ b/products/warehouse_suggestions/backend/logic/candidates/base.py
@@ -155,6 +155,7 @@ class CandidateResult:
drafts: tuple[SuggestionDraft, ...]
rejections: tuple[Rejection, ...]
skipped_reason: str | None = None
+ measurement_requests: tuple[Subject, ...] = ()
class Candidate(ABC):
diff --git a/products/warehouse_suggestions/backend/logic/candidates/materialize.py b/products/warehouse_suggestions/backend/logic/candidates/materialize.py
index 9b8da6b6ca2e..6298253bc8e1 100644
--- a/products/warehouse_suggestions/backend/logic/candidates/materialize.py
+++ b/products/warehouse_suggestions/backend/logic/candidates/materialize.py
@@ -14,6 +14,7 @@ from products.data_modeling.backend.facade.api import (
MissingDagNodeError,
SavedQueryFrequencyBounds,
definition_database,
+ dependent_saved_query_ids,
get_saved_query_columns,
propose_incremental_config,
saved_query_target_bounds,
@@ -29,7 +30,10 @@ from ...facade.enums import (
WarehouseSuggestionRefreshMode,
WarehouseSuggestionSubjectKind,
)
+from ..lineage import recomputing_consumers
from ..reads import Subject, SubjectReads
+from ..rules import MaterializeRules
+from ..view_costs import unfinished_measurement
from .base import (
DAYS_PER_MONTH,
MILLISECONDS_PER_SECOND,
@@ -43,11 +47,14 @@ from .base import (
savings_between,
stored_cost,
)
+from .unmaterialize import effective_definition
SECONDS_PER_MONTH = DAYS_PER_MONTH * 24 * 60 * 60
READ_SECONDS_PER_MONTH = "read_seconds_per_month"
READ_BYTES_PER_MONTH = "read_bytes_per_month"
HUMAN_READS_PER_MONTH = "human_reads_per_month"
+UNMEASURED_REJECTION = "its cost has not been measured yet"
+NOT_RECOMPUTED_ENOUGH_REJECTION = "it is not computed often enough for a stored copy to save half of it"
@frozen
@@ -73,6 +80,55 @@ class _Proposal:
estimate: SavingsEstimate
+@frozen
+class Recomputations:
+ consumer_runs_per_month: float
+ consumers: int
+ fastest_consumer_interval: timedelta
+
+
+@frozen
+class ReadCosts:
+ observed: QueryCost
+ saved: QueryCost
+
+
+@frozen
+class _Unmeasured:
+ recomputations: Recomputations
+
+
+class _Consumers:
+ def __init__(self, context: CandidateContext) -> None:
+ self._context = context
+
+ @cached_property
+ def _materialized_ids(self) -> frozenset[UUID]:
+ return frozenset(
+ saved_query.id for saved_query in self._context.inventory.saved_queries.values() if saved_query.materializes
+ )
+
+ @cached_property
+ def _refreshed_ids(self) -> frozenset[UUID]:
+ return frozenset(
+ saved_query_id
+ for saved_query_id in self._materialized_ids
+ if (refreshes := self._context.reads.refreshes.get(saved_query_id)) is not None
+ and refreshes.runs > 0
+ and not self._context.inventory.saved_queries[saved_query_id].refreshes_incrementally
+ )
+
+ @cached_property
+ def _dependents(self) -> Mapping[UUID, frozenset[UUID]]:
+ return dependent_saved_query_ids(self._context.team_id, list(self._context.inventory.saved_queries))
+
+ def refreshing(self, saved_query_id: UUID) -> frozenset[UUID]:
+ if not self._refreshed_ids:
+ return frozenset()
+ consumers = recomputing_consumers(saved_query_id, self._dependents, self._materialized_ids)
+ return consumers & self._refreshed_ids
+
+
class _IncrementalProposer:
"""Proposes incremental keys for one team run, building the team database at most once."""
@@ -100,33 +156,45 @@ class MaterializeCandidate(Candidate):
def evaluate(self, context: CandidateContext) -> CandidateResult:
drafts = []
rejections = []
+ unmeasured: list[tuple[Subject, Recomputations]] = []
incremental = _IncrementalProposer(context.team_id)
+ consumers = _Consumers(context)
for saved_query in context.inventory.saved_queries.values():
subject = Subject(kind=WarehouseSuggestionSubjectKind.SAVED_QUERY, id=saved_query.id)
+ if not self._is_unmaterialized(context, saved_query):
+ continue
reads = context.reads.reads_of(subject)
- if reads is None or reads.human_requests == 0 or not self._is_unmaterialized(context, saved_query):
+ consumer_ids = consumers.refreshing(saved_query.id)
+ if consumer_ids:
+ outcome = self._propose_recomputed(context, saved_query, reads, consumer_ids, incremental)
+ elif reads is None or reads.human_requests == 0:
+ continue
+ else:
+ outcome = self._propose(context, saved_query, reads, incremental)
+ if isinstance(outcome, _Unmeasured):
+ unmeasured.append((subject, outcome.recomputations))
+ rejections.append(Rejection(subject=subject, reason=UNMEASURED_REJECTION))
continue
- outcome = self._propose(context, saved_query, reads, incremental)
if isinstance(outcome, str):
rejections.append(Rejection(subject=subject, reason=outcome))
continue
+ score_inputs = {
+ "seconds_saved_per_month": outcome.estimate.seconds_saved,
+ "bytes_saved_per_month": outcome.estimate.bytes_saved,
+ READ_SECONDS_PER_MONTH: outcome.estimate.read_seconds,
+ READ_BYTES_PER_MONTH: outcome.estimate.read_bytes,
+ "refresh_interval_seconds": outcome.estimate.interval.total_seconds(),
+ }
+ if not consumer_ids and reads is not None:
+ score_inputs[HUMAN_READS_PER_MONTH] = reads.human_reads * context.month_scale
drafts.append(
- self.draft(
- context,
- subject,
- outcome.payload,
- score=outcome.estimate.score,
- score_inputs={
- "seconds_saved_per_month": outcome.estimate.seconds_saved,
- "bytes_saved_per_month": outcome.estimate.bytes_saved,
- READ_SECONDS_PER_MONTH: outcome.estimate.read_seconds,
- READ_BYTES_PER_MONTH: outcome.estimate.read_bytes,
- HUMAN_READS_PER_MONTH: reads.human_reads * context.month_scale,
- "refresh_interval_seconds": outcome.estimate.interval.total_seconds(),
- },
- )
+ self.draft(context, subject, outcome.payload, score=outcome.estimate.score, score_inputs=score_inputs)
)
- return CandidateResult(drafts=tuple(drafts), rejections=tuple(rejections))
+ return CandidateResult(
+ drafts=tuple(drafts),
+ rejections=tuple(rejections),
+ measurement_requests=_most_recomputed(unmeasured, context.rules.measurement.max_per_team),
+ )
def asset_outcome(
self, context: CandidateContext, subject: Subject, accepted_at: datetime | None
@@ -221,6 +289,63 @@ class MaterializeCandidate(Candidate):
estimate=estimate,
)
+ def _propose_recomputed(
+ self,
+ context: CandidateContext,
+ saved_query: SavedQueryDefinition,
+ reads: SubjectReads | None,
+ consumer_ids: frozenset[UUID],
+ incremental: _IncrementalProposer,
+ ) -> _Proposal | _Unmeasured | str:
+ if context.accepted_recently(WarehouseSuggestionKind.UNMATERIALIZE, saved_query.id):
+ return f"was unmaterialized on a suggestion in the last {context.rules.lifecycle.flip_cooldown_days} days"
+ recomputations = count_recomputations(context, consumer_ids)
+ ceiling = min(context.rules.refresh.max_interval, recomputations.fastest_consumer_interval)
+ reads_qualify = (
+ reads is not None and reads.human_requests > 0 and self._read_floor_rejection(context, reads) is None
+ )
+ if not reads_qualify and not recomputed_often_enough(context, recomputations, ceiling):
+ return NOT_RECOMPUTED_ENOUGH_REJECTION
+ frequency = saved_query_target_bounds(context.team_id, saved_query.id)
+ if frequency is None:
+ capture_exception(MissingDagNodeError(f"Saved query {saved_query.id} has no DAG node"))
+ return "it has no node in a DAG"
+ intervals = sorted(interval for interval in frequency.bounds.allowed if interval <= ceiling)
+ if not intervals:
+ return "no refresh interval keeps up with the views that read it"
+ interval = intervals[-1]
+ if not reads_qualify and not recomputed_often_enough(context, recomputations, interval):
+ return NOT_RECOMPUTED_ENOUGH_REJECTION
+ cost = evaluation_cost(context, saved_query, reads)
+ if isinstance(cost, str):
+ return cost
+ if cost is None:
+ return _Unmeasured(recomputations=recomputations)
+ estimate = recomputation_savings(context, reads, recomputations, interval, cost)
+ if not estimate.clears_a_floor:
+ return "savings stay under the floors at the interval the views that read it need"
+ sources = _classify_sources(context, saved_query.id, frequency)
+ freshness_today = int(frequency.max_data_age.total_seconds()) or None
+ interval_seconds = int(interval.total_seconds())
+ payload = MaterializePayload(
+ subject_name=saved_query.name,
+ refresh_interval_seconds=interval_seconds,
+ saves_seconds_per_month=estimate.seconds_saved,
+ saves_bytes_per_month=estimate.bytes_saved,
+ freshness_today_seconds=freshness_today,
+ freshness_after_seconds=interval_seconds + (freshness_today or 0),
+ live_sources=sources.live,
+ unknown_sources=sources.unknown,
+ refresh_interval_basis=WarehouseSuggestionRefreshIntervalBasis.CONSUMER_REFRESH,
+ recomputed_by_views=recomputations.consumers,
+ )
+ return _Proposal(
+ payload=_with_refresh_mode(
+ payload, incremental.config_for(saved_query), context.rules.materialize.incremental_lookback_seconds
+ ),
+ estimate=estimate,
+ )
+
def _read_floor_rejection(self, context: CandidateContext, reads: SubjectReads) -> str | None:
rules = context.rules.materialize
min_days = context.scaled_floor(rules.min_days)
@@ -290,30 +415,121 @@ def refresh_cost(context: CandidateContext, saved_query_id: UUID, reads: Subject
def estimate_savings(
context: CandidateContext, reads: SubjectReads, interval: timedelta, refresh: QueryCost
) -> SavingsEstimate:
- rules = context.rules.materialize
+ refreshes_per_month = SECONDS_PER_MONTH / interval.total_seconds()
+ read_costs = read_costs_of(context, reads)
+ return _estimate(
+ context.rules.materialize,
+ interval,
+ cost_without=read_costs.observed,
+ saved=QueryCost(
+ seconds=read_costs.saved.seconds - refreshes_per_month * refresh.seconds,
+ read_bytes=read_costs.saved.read_bytes - refreshes_per_month * refresh.read_bytes,
+ ),
+ )
+
+
+def read_costs_of(context: CandidateContext, reads: SubjectReads | None) -> ReadCosts:
+ if reads is None:
+ return ReadCosts(observed=QueryCost(seconds=0, read_bytes=0), saved=QueryCost(seconds=0, read_bytes=0))
scale = context.month_scale
+ observed = QueryCost(
+ seconds=reads.human_duration_ms / MILLISECONDS_PER_SECOND * scale, read_bytes=reads.human_read_bytes * scale
+ )
+ computed_on_read = QueryCost(
+ seconds=reads.human_reads * reads.alone_duration_ms_median / MILLISECONDS_PER_SECOND * scale,
+ read_bytes=reads.human_reads * reads.alone_read_bytes_median * scale,
+ )
+ return ReadCosts(
+ observed=observed,
+ saved=QueryCost(
+ seconds=min(observed.seconds, computed_on_read.seconds),
+ read_bytes=min(observed.read_bytes, computed_on_read.read_bytes),
+ ),
+ )
+
+
+def count_recomputations(context: CandidateContext, consumer_ids: frozenset[UUID]) -> Recomputations:
+ consumer_runs = [context.reads.refreshes[consumer_id].runs for consumer_id in consumer_ids]
+ return Recomputations(
+ consumer_runs_per_month=sum(consumer_runs) * context.month_scale,
+ consumers=len(consumer_ids),
+ fastest_consumer_interval=timedelta(days=context.days_observed) / max(consumer_runs),
+ )
+
+
+def recomputed_often_enough(context: CandidateContext, recomputations: Recomputations, interval: timedelta) -> bool:
refreshes_per_month = SECONDS_PER_MONTH / interval.total_seconds()
- body_seconds = reads.alone_duration_ms_median / MILLISECONDS_PER_SECOND
- read_seconds = reads.human_duration_ms / MILLISECONDS_PER_SECOND * scale
- read_bytes = reads.human_read_bytes * scale
- seconds_saved = min(read_seconds, reads.human_reads * body_seconds * scale) - refreshes_per_month * refresh.seconds
- bytes_saved = (
- min(read_bytes, reads.human_reads * reads.alone_read_bytes_median * scale)
- - refreshes_per_month * refresh.read_bytes
+ runs = recomputations.consumer_runs_per_month
+ return runs - refreshes_per_month >= context.rules.materialize.min_saved_share * runs
+
+
+def evaluation_cost(
+ context: CandidateContext, saved_query: SavedQueryDefinition, reads: SubjectReads | None
+) -> QueryCost | str | None:
+ definition = effective_definition(context.team_id, saved_query)
+ if reads is not None and reads.last_measured_at is not None:
+ if definition.changed_at is None or definition.changed_at < reads.last_measured_at:
+ return QueryCost(
+ seconds=reads.measured_duration_ms / MILLISECONDS_PER_SECOND, read_bytes=reads.measured_read_bytes
+ )
+ unfinished = unfinished_measurement(context.team_id, saved_query.id)
+ if unfinished is None or unfinished.fingerprint != definition.fingerprint:
+ return None
+ if unfinished.failed:
+ return "its cost could not be measured"
+ return QueryCost(seconds=unfinished.seconds, read_bytes=0)
+
+
+def recomputation_savings(
+ context: CandidateContext,
+ reads: SubjectReads | None,
+ recomputations: Recomputations,
+ interval: timedelta,
+ cost: QueryCost,
+) -> SavingsEstimate:
+ runs = recomputations.consumer_runs_per_month
+ runs_saved = runs - SECONDS_PER_MONTH / interval.total_seconds()
+ read_costs = read_costs_of(context, reads)
+ return _estimate(
+ context.rules.materialize,
+ interval,
+ cost_without=QueryCost(
+ seconds=runs * cost.seconds + read_costs.observed.seconds,
+ read_bytes=runs * cost.read_bytes + read_costs.observed.read_bytes,
+ ),
+ saved=QueryCost(
+ seconds=runs_saved * cost.seconds + read_costs.saved.seconds,
+ read_bytes=runs_saved * cost.read_bytes + read_costs.saved.read_bytes,
+ ),
+ )
+
+
+def _estimate(
+ rules: MaterializeRules, interval: timedelta, *, cost_without: QueryCost, saved: QueryCost
+) -> SavingsEstimate:
+ clears_time = (
+ saved.seconds >= rules.min_seconds_saved and saved.seconds >= rules.min_saved_share * cost_without.seconds
+ )
+ clears_bytes = (
+ saved.read_bytes >= rules.min_bytes_saved
+ and saved.read_bytes >= rules.min_saved_share * cost_without.read_bytes
)
- clears_time = seconds_saved >= rules.min_seconds_saved and seconds_saved >= rules.min_saved_share * read_seconds
- clears_bytes = bytes_saved >= rules.min_bytes_saved and bytes_saved >= rules.min_saved_share * read_bytes
return SavingsEstimate(
interval=interval,
- read_seconds=read_seconds,
- read_bytes=read_bytes,
- seconds_saved=seconds_saved,
- bytes_saved=bytes_saved,
+ read_seconds=cost_without.seconds,
+ read_bytes=cost_without.read_bytes,
+ seconds_saved=saved.seconds,
+ bytes_saved=saved.read_bytes,
clears_a_floor=clears_time or clears_bytes,
- score=max(seconds_saved / rules.min_seconds_saved, bytes_saved / rules.min_bytes_saved),
+ score=max(saved.seconds / rules.min_seconds_saved, saved.read_bytes / rules.min_bytes_saved),
)
+def _most_recomputed(unmeasured: list[tuple[Subject, Recomputations]], limit: int) -> tuple[Subject, ...]:
+ ranked = sorted(unmeasured, key=lambda pair: pair[1].consumer_runs_per_month, reverse=True)
+ return tuple(subject for subject, _ in ranked[:limit])
+
+
def _read_cost_before_at(score_inputs: Mapping[str, float], reads_per_month: float) -> QueryCost | None:
cost_before = stored_cost(score_inputs, READ_SECONDS_PER_MONTH, READ_BYTES_PER_MONTH)
reads_before = score_inputs.get(HUMAN_READS_PER_MONTH)
diff --git a/products/warehouse_suggestions/backend/logic/job.py b/products/warehouse_suggestions/backend/logic/job.py
index ddf4f05e25b7..4df998cd425f 100644
--- a/products/warehouse_suggestions/backend/logic/job.py
+++ b/products/warehouse_suggestions/backend/logic/job.py
@@ -10,7 +10,6 @@ from posthog.dataclasses import frozen
from posthog.models.team import Team
from posthog.models.team.extensions import get_or_create_team_extension
-from ..facade.contracts import SuggestionDraft
from ..facade.enums import WarehouseSuggestionKind, WarehouseSuggestionStatus
from ..models import WarehouseSuggestion, WarehouseSuggestionTeamConfig
from .analytics import report_candidates
@@ -18,7 +17,7 @@ from .candidates.base import CandidateContext, CandidateResult
from .candidates.registry import evaluate_candidates
from .inventory import load_inventory
from .lifecycle import FLIP_KINDS, LifecycleResult, apply_run
-from .reads import ReadWindow, RollupDays, TeamReads, read_rollup_days, read_team_reads
+from .reads import ReadWindow, RollupDays, Subject, TeamReads, read_rollup_days, read_team_reads
from .rules import RULES, EligibilityRules, Rules
MAINTAINED_STATUSES = (WarehouseSuggestionStatus.PROPOSED, WarehouseSuggestionStatus.ACCEPTED)
@@ -36,6 +35,7 @@ class TeamRunResult:
status: TeamRunStatus
drafts: int = 0
lifecycle: LifecycleResult | None = None
+ measurement_requests: tuple[Subject, ...] = ()
def run_team(
@@ -59,12 +59,19 @@ def run_team(
_record_run(_locked_config(team_id), reads, eligible=False, now=now)
return TeamRunResult(team_id=team_id, status=status)
context = build_context(team, reads, run_id=run_id, rules=rules, now=now)
- drafts = _evaluate_and_report(context) if eligible else []
+ results = _evaluate_and_report(context) if eligible else {}
+ drafts = [draft for result in results.values() for draft in result.drafts]
with transaction.atomic():
config = _locked_config(team_id)
_record_run(config, reads, eligible=eligible, now=now)
lifecycle = apply_run(context, team, drafts, now, surface=eligible and config.paused_reason is None)
- return TeamRunResult(team_id=team_id, status=status, drafts=len(drafts), lifecycle=lifecycle)
+ return TeamRunResult(
+ team_id=team_id,
+ status=status,
+ drafts=len(drafts),
+ lifecycle=lifecycle,
+ measurement_requests=tuple(subject for result in results.values() for subject in result.measurement_requests),
+ )
def explain_team(
@@ -77,11 +84,11 @@ def explain_team(
return context, evaluate_candidates(context)
-def _evaluate_and_report(context: CandidateContext) -> list[SuggestionDraft]:
+def _evaluate_and_report(context: CandidateContext) -> dict[WarehouseSuggestionKind, CandidateResult]:
results = evaluate_candidates(context)
for kind, result in results.items():
report_candidates(kind, len(result.drafts))
- return [draft for result in results.values() for draft in result.drafts]
+ return results
def is_eligible(reads: TeamReads, rules: EligibilityRules) -> bool:
diff --git a/products/warehouse_suggestions/backend/logic/lineage.py b/products/warehouse_suggestions/backend/logic/lineage.py
new file mode 100644
index 000000000000..80bbf7d7fccf
--- /dev/null
+++ b/products/warehouse_suggestions/backend/logic/lineage.py
@@ -0,0 +1,20 @@
+from collections.abc import Collection, Mapping
+from uuid import UUID
+
+
+def recomputing_consumers(
+ view_id: UUID, dependents: Mapping[UUID, Collection[UUID]], materialized_ids: Collection[UUID]
+) -> frozenset[UUID]:
+ consumers: set[UUID] = set()
+ visited: set[UUID] = {view_id}
+ frontier = [view_id]
+ while frontier:
+ for dependent_id in dependents.get(frontier.pop(), ()):
+ if dependent_id in visited:
+ continue
+ visited.add(dependent_id)
+ if dependent_id in materialized_ids:
+ consumers.add(dependent_id)
+ else:
+ frontier.append(dependent_id)
+ return frozenset(consumers)
diff --git a/products/warehouse_suggestions/backend/logic/reads.py b/products/warehouse_suggestions/backend/logic/reads.py
index 5f6f53bf4ef5..2e8785ed0991 100644
--- a/products/warehouse_suggestions/backend/logic/reads.py
+++ b/products/warehouse_suggestions/backend/logic/reads.py
@@ -1,6 +1,7 @@
from collections import defaultdict
from collections.abc import Mapping
from datetime import UTC, date, datetime, time, timedelta
+from math import isfinite
from typing import Any
from uuid import UUID
@@ -15,6 +16,8 @@ from .rules import Rules, Surface, SurfaceRules, TrafficRules
READS_QUERY_SETTINGS = {"max_execution_time": 30, "max_threads": 2}
FIRST_QUANTILE = 1
SECOND_QUANTILE = 2
+MEASUREMENT_FEATURE = Feature.WAREHOUSE_SUGGESTIONS
+UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
DAYS_FILTER = """day >= %(window_start)s
AND day < %(window_end)s"""
@@ -77,6 +80,10 @@ class SubjectReads:
alone_read_bytes_p90: float
last_read_at: datetime
requests_by_surface: Mapping[Surface, int]
+ measured_reads: int = 0
+ measured_duration_ms: float = 0.0
+ measured_read_bytes: float = 0.0
+ last_measured_at: datetime | None = None
@property
def surfaces(self) -> frozenset[Surface]:
@@ -140,6 +147,7 @@ def read_team_reads(team_id: int, window: ReadWindow, rules: Rules, rollup_days:
"read": ReadKind.READ.value,
"refresh": ReadKind.REFRESH.value,
"saved_query": WarehouseSuggestionSubjectKind.SAVED_QUERY.value,
+ "measurement_feature": MEASUREMENT_FEATURE.value,
}
human_sql, human_params = human_condition(rules.traffic)
surface_sql, surface_params = surface_expression(rules.surfaces)
@@ -227,14 +235,21 @@ SELECT
sumIf(read_count, is_human),
sumIf(duration_ms_sum, is_human),
sumIf(read_bytes_sum, is_human),
- sumIf(read_count, read_alone),
- quantilesMergeIf(0.5, 0.9)(duration_ms_quantiles, read_alone)[{FIRST_QUANTILE}],
- quantilesMergeIf(0.5, 0.9)(read_bytes_quantiles, read_alone)[{FIRST_QUANTILE}],
- quantilesMergeIf(0.5, 0.9)(duration_ms_quantiles, read_alone)[{SECOND_QUANTILE}],
- quantilesMergeIf(0.5, 0.9)(read_bytes_quantiles, read_alone)[{SECOND_QUANTILE}],
- toTimeZone(max(max_event_time), 'UTC')
+ sumIf(read_count, read_alone AND NOT is_measurement),
+ quantilesMergeIf(0.5, 0.9)(duration_ms_quantiles, read_alone AND NOT is_measurement)[{FIRST_QUANTILE}],
+ quantilesMergeIf(0.5, 0.9)(read_bytes_quantiles, read_alone AND NOT is_measurement)[{FIRST_QUANTILE}],
+ quantilesMergeIf(0.5, 0.9)(duration_ms_quantiles, read_alone AND NOT is_measurement)[{SECOND_QUANTILE}],
+ quantilesMergeIf(0.5, 0.9)(read_bytes_quantiles, read_alone AND NOT is_measurement)[{SECOND_QUANTILE}],
+ toTimeZone(max(max_event_time), 'UTC'),
+ sumIf(read_count, is_measurement),
+ argMaxIf(duration_ms_sum / read_count, max_event_time, is_measurement),
+ argMaxIf(read_bytes_sum / read_count, max_event_time, is_measurement),
+ toTimeZone(maxIf(max_event_time, is_measurement), 'UTC')
FROM (
- SELECT *, {{human}} AS is_human
+ SELECT
+ *,
+ {{human}} AS is_human,
+ lc_feature = %(measurement_feature)s AND read_alone AS is_measurement
FROM {WAREHOUSE_OBJECT_READS_DAILY_TABLE}
WHERE {WINDOW_FILTER}
AND read_kind = %(read)s
@@ -321,10 +336,18 @@ def _subject_reads(
alone_read_bytes_p90=float(row[16]),
last_read_at=row[17],
requests_by_surface=surface_counts.get(subject, {}),
+ measured_reads=row[18],
+ measured_duration_ms=_finite(row[19]),
+ measured_read_bytes=_finite(row[20]),
+ last_measured_at=row[21] if row[21] > UNIX_EPOCH else None,
)
return reads
+def _finite(value: float) -> float:
+ return float(value) if isfinite(value) else 0.0
+
+
def _refreshes(params: Mapping[str, Any], team_id: int) -> dict[UUID, SubjectRefreshes]:
refreshes: dict[UUID, SubjectRefreshes] = {}
for subject_id, runs, duration_ms, read_bytes in _execute(REFRESHES_SQL, params, team_id):
diff --git a/products/warehouse_suggestions/backend/logic/rules.py b/products/warehouse_suggestions/backend/logic/rules.py
index 80cbd21124b6..a6b15f78415f 100644
--- a/products/warehouse_suggestions/backend/logic/rules.py
+++ b/products/warehouse_suggestions/backend/logic/rules.py
@@ -105,6 +105,14 @@ class MaterializeRules:
incremental_lookback_seconds: int
+@frozen
+class MeasurementRules:
+ max_per_team: int
+ max_per_run: int
+ max_seconds: int
+ retry_after_days: int
+
+
@frozen
class UnmaterializeRules:
min_runs: int
@@ -143,6 +151,7 @@ class Rules:
certify: CertifyRules
deprecate: DeprecateRules
materialize: MaterializeRules
+ measurement: MeasurementRules
unmaterialize: UnmaterializeRules
refresh: RefreshRules
lifecycle: LifecycleRules
@@ -214,6 +223,7 @@ RULES = Rules(
min_saved_share=0.5,
incremental_lookback_seconds=int(timedelta(days=3).total_seconds()),
),
+ measurement=MeasurementRules(max_per_team=3, max_per_run=50, max_seconds=180, retry_after_days=7),
unmaterialize=UnmaterializeRules(
min_runs=3,
min_seconds_saved=600.0,
diff --git a/products/warehouse_suggestions/backend/logic/view_costs.py b/products/warehouse_suggestions/backend/logic/view_costs.py
new file mode 100644
index 000000000000..888a52b4c84e
--- /dev/null
+++ b/products/warehouse_suggestions/backend/logic/view_costs.py
@@ -0,0 +1,103 @@
+from datetime import timedelta
+from enum import StrEnum
+from uuid import UUID
+
+from django.core.cache import cache
+
+from posthog.hogql import ast
+from posthog.hogql.constants import HogQLGlobalSettings
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.clickhouse.query_tagging import Product, tag_queries
+from posthog.clickhouse.workload import Workload
+from posthog.dataclasses import frozen
+from posthog.errors import ClickHouseQueryTimeOut
+from posthog.exceptions_capture import capture_exception
+from posthog.models.team import Team
+
+from products.data_modeling.backend.facade.api import get_saved_query_columns, saved_query_definitions
+
+from .candidates.unmaterialize import effective_definition
+from .reads import MEASUREMENT_FEATURE
+from .rules import MeasurementRules
+
+MEASUREMENT_QUERY_NAME = "warehouse_suggestions_view_cost"
+UNFINISHED_KEY_PREFIX = "warehouse_suggestions:unfinished_view_cost"
+FAILED_MEASUREMENT = 0
+
+
+class MeasurementOutcome(StrEnum):
+ MEASURED = "measured"
+ CAPPED = "capped"
+ FAILED = "failed"
+ GONE = "gone"
+
+
+@frozen
+class UnfinishedMeasurement:
+ seconds: int
+ fingerprint: str
+
+ @property
+ def failed(self) -> bool:
+ return self.seconds == FAILED_MEASUREMENT
+
+
+def unfinished_measurement(team_id: int, saved_query_id: UUID) -> UnfinishedMeasurement | None:
+ remembered = cache.get(_unfinished_key(team_id, saved_query_id))
+ if not isinstance(remembered, dict):
+ return None
+ return UnfinishedMeasurement(seconds=remembered["seconds"], fingerprint=remembered["fingerprint"])
+
+
+def measure_view_cost(team_id: int, saved_query_id: UUID, rules: MeasurementRules) -> MeasurementOutcome:
+ definitions = saved_query_definitions(team_id, ids=[saved_query_id])
+ if not definitions:
+ return MeasurementOutcome.GONE
+ definition = definitions[0]
+ fingerprint = effective_definition(team_id, definition).fingerprint
+ column_names = list(get_saved_query_columns(team_id, saved_query_id))
+ if not column_names:
+ _remember_unfinished(team_id, saved_query_id, rules, seconds=FAILED_MEASUREMENT, fingerprint=fingerprint)
+ return MeasurementOutcome.FAILED
+ tag_queries(
+ product=Product.DATA_MODELING, feature=MEASUREMENT_FEATURE, team_id=team_id, name=MEASUREMENT_QUERY_NAME
+ )
+ try:
+ execute_hogql_query(
+ query=full_read_query(definition.name, column_names),
+ team=Team.objects.get(id=team_id),
+ workload=Workload.OFFLINE,
+ settings=HogQLGlobalSettings(max_execution_time=rules.max_seconds),
+ bypass_warehouse_access_control=True,
+ )
+ except ClickHouseQueryTimeOut:
+ _remember_unfinished(team_id, saved_query_id, rules, seconds=rules.max_seconds, fingerprint=fingerprint)
+ return MeasurementOutcome.CAPPED
+ except Exception as error:
+ capture_exception(error)
+ _remember_unfinished(team_id, saved_query_id, rules, seconds=FAILED_MEASUREMENT, fingerprint=fingerprint)
+ return MeasurementOutcome.FAILED
+ return MeasurementOutcome.MEASURED
+
+
+def full_read_query(view_name: str, column_names: list[str]) -> ast.SelectQuery:
+ every_column = ast.Call(name="tuple", args=[ast.Field(chain=[name]) for name in column_names])
+ row_hash = ast.Call(name="cityHash64", args=[ast.Call(name="toString", args=[every_column])])
+ return ast.SelectQuery(
+ select=[ast.Call(name="count", args=[]), ast.Call(name="sum", args=[row_hash])],
+ select_from=ast.JoinExpr(table=ast.Field(chain=[view_name])),
+ )
+
+
+def _remember_unfinished(
+ team_id: int, saved_query_id: UUID, rules: MeasurementRules, *, seconds: int, fingerprint: str
+) -> None:
+ timeout = int(timedelta(days=rules.retry_after_days).total_seconds())
+ cache.set(
+ _unfinished_key(team_id, saved_query_id), {"seconds": seconds, "fingerprint": fingerprint}, timeout=timeout
+ )
+
+
+def _unfinished_key(team_id: int, saved_query_id: UUID) -> str:
+ return f"{UNFINISHED_KEY_PREFIX}:{team_id}:{saved_query_id}"
diff --git a/products/warehouse_suggestions/backend/presentation/serializers.py b/products/warehouse_suggestions/backend/presentation/serializers.py
index 7566d3383f8a..3a939f28945c 100644
--- a/products/warehouse_suggestions/backend/presentation/serializers.py
+++ b/products/warehouse_suggestions/backend/presentation/serializers.py
@@ -97,6 +97,7 @@ class MaterializePayloadSerializer(DataclassSerializer):
allow_null=True,
help_text="Why the interval is not shorter. source_sync: the view's sources sync no faster. "
"fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. "
+ "consumer_refresh: the materialized views that read it refresh no more often. "
"savings: a shorter interval would cost more than it saves. Null on suggestions made before this was "
"recorded.",
)
@@ -125,6 +126,10 @@ class MaterializePayloadSerializer(DataclassSerializer):
"help_text": "How far before the last refresh's highest incremental_key value each refresh reads "
"again, in seconds. The last incremental_key value is always read again."
},
+ "recomputed_by_views": {
+ "help_text": "How many materialized views compute this view again on every refresh, because it "
+ "is not stored. 0 when the saving comes only from people's reads."
+ },
}
diff --git a/products/warehouse_suggestions/backend/temporal/__init__.py b/products/warehouse_suggestions/backend/temporal/__init__.py
index 45b1a20aaa21..849a51226f4d 100644
--- a/products/warehouse_suggestions/backend/temporal/__init__.py
+++ b/products/warehouse_suggestions/backend/temporal/__init__.py
@@ -5,11 +5,15 @@ from typing import Any
from posthog.temporal.common.base import PostHogWorkflow
-from .activities import generate_warehouse_suggestions, get_warehouse_suggestion_team_batches
+from .activities import generate_warehouse_suggestions, get_warehouse_suggestion_team_batches, measure_view_costs
from .workflows import WarehouseSuggestionsWorkflow
WORKFLOWS: list[type[PostHogWorkflow]] = [WarehouseSuggestionsWorkflow]
-ACTIVITIES: list[Callable[..., Any]] = [get_warehouse_suggestion_team_batches, generate_warehouse_suggestions]
+ACTIVITIES: list[Callable[..., Any]] = [
+ get_warehouse_suggestion_team_batches,
+ generate_warehouse_suggestions,
+ measure_view_costs,
+]
__all__ = ["ACTIVITIES", "WORKFLOWS"]
diff --git a/products/warehouse_suggestions/backend/temporal/activities.py b/products/warehouse_suggestions/backend/temporal/activities.py
index e3e715e38b55..1593586631c2 100644
--- a/products/warehouse_suggestions/backend/temporal/activities.py
+++ b/products/warehouse_suggestions/backend/temporal/activities.py
@@ -1,5 +1,6 @@
from collections import Counter
from itertools import batched
+from uuid import UUID
from django.utils import timezone
@@ -19,7 +20,8 @@ from ..logic.flags import is_warehouse_suggestions_enabled
from ..logic.job import TeamRunStatus, run_team, team_ids_with_suggestions_to_maintain
from ..logic.reads import ReadWindow, read_rollup_days
from ..logic.rules import RULES
-from .contracts import BatchOutcome, WarehouseSuggestionsInputs
+from ..logic.view_costs import MeasurementOutcome, measure_view_cost
+from .contracts import BatchOutcome, ViewCostRequest, WarehouseSuggestionsInputs
LOGGER = get_logger(__name__)
FULL_ROLLOUT = 1.0
@@ -53,18 +55,35 @@ def run_batch(team_ids: list[int], run_id: str) -> BatchOutcome:
today = timezone.now().date()
rollup_days = read_rollup_days(ReadWindow.ending(today, RULES))
outcomes: Counter[str] = Counter()
+ view_costs_to_measure: list[ViewCostRequest] = []
for team_id in team_ids:
try:
- outcomes[run_team(team_id, run_id=run_id, today=today, rollup_days=rollup_days).status] += 1
+ result = run_team(team_id, run_id=run_id, today=today, rollup_days=rollup_days)
except Exception:
LOGGER.exception("warehouse_suggestions.team_failed", team_id=team_id)
outcomes["failed"] += 1
+ continue
+ outcomes[result.status] += 1
+ view_costs_to_measure.extend(
+ ViewCostRequest(team_id=team_id, saved_query_id=str(subject.id)) for subject in result.measurement_requests
+ )
return BatchOutcome(
processed=outcomes[TeamRunStatus.PROCESSED],
not_eligible=outcomes[TeamRunStatus.NOT_ELIGIBLE],
disabled=outcomes[TeamRunStatus.DISABLED],
failed=outcomes["failed"],
+ view_costs_to_measure=view_costs_to_measure[: RULES.measurement.max_per_run],
+ )
+
+
+def measure_view_costs_sync(requests: list[ViewCostRequest]) -> int:
+ outcomes: Counter[MeasurementOutcome] = Counter()
+ for request in requests:
+ outcomes[measure_view_cost(request.team_id, UUID(request.saved_query_id), RULES.measurement)] += 1
+ LOGGER.info(
+ "warehouse_suggestions.view_costs_measured", **{outcome.value: count for outcome, count in outcomes.items()}
)
+ return outcomes.total()
def _teams_to_run() -> set[int]:
@@ -87,3 +106,9 @@ async def get_warehouse_suggestion_team_batches(inputs: WarehouseSuggestionsInpu
async def generate_warehouse_suggestions(team_ids: list[int], run_id: str) -> BatchOutcome:
async with Heartbeater():
return await database_sync_to_async_pool(run_batch)(team_ids, run_id)
+
+
+async def measure_view_costs(requests: list[ViewCostRequest]) -> int:
+ async with Heartbeater():
+ return await database_sync_to_async_pool(measure_view_costs_sync)(requests)
diff --git a/products/warehouse_suggestions/backend/temporal/contracts.py b/products/warehouse_suggestions/backend/temporal/contracts.py
index c6c55112120e..4c4d36924e0a 100644
--- a/products/warehouse_suggestions/backend/temporal/contracts.py
+++ b/products/warehouse_suggestions/backend/temporal/contracts.py
@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
WAREHOUSE_SUGGESTIONS_WORKFLOW_NAME = "warehouse-suggestions"
@@ -11,9 +11,16 @@ class WarehouseSuggestionsInputs:
team_ids: list[int] | None = None
+@dataclass(frozen=True, kw_only=True)
+class ViewCostRequest:
+ team_id: int
+ saved_query_id: str
+
+
@dataclass(frozen=True, kw_only=True)
class BatchOutcome:
processed: int = 0
not_eligible: int = 0
disabled: int = 0
failed: int = 0
+ view_costs_to_measure: list[ViewCostRequest] = field(default_factory=list)
diff --git a/products/warehouse_suggestions/backend/temporal/workflows.py b/products/warehouse_suggestions/backend/temporal/workflows.py
index bdecd6cf4657..8a8ce6f14b72 100644
--- a/products/warehouse_suggestions/backend/temporal/workflows.py
+++ b/products/warehouse_suggestions/backend/temporal/workflows.py
@@ -7,9 +7,15 @@ from temporalio.common import RetryPolicy
from posthog.temporal.common.base import PostHogWorkflow
-from .activities import generate_warehouse_suggestions, get_warehouse_suggestion_team_batches
+from ..logic.rules import RULES
+from .activities import generate_warehouse_suggestions, get_warehouse_suggestion_team_batches, measure_view_costs
from .contracts import WAREHOUSE_SUGGESTIONS_WORKFLOW_NAME, BatchOutcome, WarehouseSuggestionsInputs
+MEASURE_VIEW_COSTS_PATCH = "measure-view-costs"
+MEASUREMENT_MARGIN = dt.timedelta(minutes=10)
+MEASUREMENT_TIMEOUT = (
+ dt.timedelta(seconds=RULES.measurement.max_per_run * RULES.measurement.max_seconds) + MEASUREMENT_MARGIN
+)
RETRY_POLICY = RetryPolicy(
maximum_attempts=3,
initial_interval=dt.timedelta(seconds=30),
@@ -55,4 +61,15 @@ class WarehouseSuggestionsWorkflow(PostHogWorkflow):
for field in dataclasses.fields(BatchOutcome)
}
)
+ total = dataclasses.replace(
+ total, view_costs_to_measure=total.view_costs_to_measure[: RULES.measurement.max_per_run]
+ )
+ if total.view_costs_to_measure and workflow.patched(MEASURE_VIEW_COSTS_PATCH):
+ await workflow.execute_activity(
+ measure_view_costs,
+ total.view_costs_to_measure,
+ start_to_close_timeout=MEASUREMENT_TIMEOUT,
+ heartbeat_timeout=dt.timedelta(minutes=5),
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
return total
diff --git a/products/warehouse_suggestions/backend/tests/test_candidates.py b/products/warehouse_suggestions/backend/tests/test_candidates.py
index 85b1d231ceed..9c55771f9907 100644
--- a/products/warehouse_suggestions/backend/tests/test_candidates.py
+++ b/products/warehouse_suggestions/backend/tests/test_candidates.py
@@ -1,5 +1,6 @@
from dataclasses import replace
from datetime import timedelta
+from types import SimpleNamespace
from typing import Any
from uuid import uuid4
@@ -40,9 +41,13 @@ from products.warehouse_suggestions.backend.logic.candidates.materialize import
interval_basis,
refresh_cost,
)
-from products.warehouse_suggestions.backend.logic.candidates.unmaterialize import UnmaterializeCandidate
+from products.warehouse_suggestions.backend.logic.candidates.unmaterialize import (
+ EffectiveDefinition,
+ UnmaterializeCandidate,
+)
from products.warehouse_suggestions.backend.logic.reads import RefreshRun, SubjectRefreshes
from products.warehouse_suggestions.backend.logic.rules import RULES, Surface
+from products.warehouse_suggestions.backend.logic.view_costs import UnfinishedMeasurement
from products.warehouse_suggestions.backend.tests.factories import (
busy_reads,
context,
@@ -571,3 +576,116 @@ class TestMaterializeAfterUnmaterialize(SimpleTestCase):
(),
["was unmaterialized on a suggestion in the last 30 days"],
)
+
+
+MATERIALIZE_MODULE = "products.warehouse_suggestions.backend.logic.candidates.materialize"
+FIRST_CONSUMER, SECOND_CONSUMER = uuid4(), uuid4()
+CURRENT_DEFINITION, EARLIER_DEFINITION = "current", "earlier"
+TIME_LIMIT = RULES.measurement.max_seconds
+EVERY_BUCKET_ALLOWED = SimpleNamespace(
+ bounds=SimpleNamespace(allowed=(timedelta(minutes=15), HOUR, timedelta(hours=6), DAY), floor=None),
+ max_data_age=timedelta(0),
+ best_effort_source_ids=set(),
+ identities={},
+)
+UNREAD = {
+ "human_requests": 0,
+ "human_reads": 0,
+ "human_days": 0,
+ "human_duration_ms": 0,
+ "human_read_bytes": 0,
+ "alone_reads": 0,
+}
+CHEAP_PREVIEWS = {
+ "human_requests": HOURLY_RUNS,
+ "human_reads": HOURLY_RUNS,
+ "human_duration_ms": HOURLY_RUNS * 500,
+ "human_read_bytes": HOURLY_RUNS * 10**6,
+ "alone_duration_ms_median": 500.0,
+ "alone_read_bytes_median": float(10**6),
+}
+
+
+class TestMaterializeRecomputedUpstream(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("one_consumer_refreshes_it_as_often_as_a_stored_copy_would", 1, 0, UNREAD, 3, None, None, False, False),
+ ("two_consumers_with_a_measured_cost", 2, 0, UNREAD, 3, None, None, True, False),
+ ("two_consumers_and_no_measurement", 2, 0, UNREAD, None, None, None, False, True),
+ ("measured_before_the_last_edit", 2, 0, UNREAD, 3, 1, None, False, True),
+ ("measurement_hit_the_time_limit", 2, 0, UNREAD, None, None, (TIME_LIMIT, CURRENT_DEFINITION), True, False),
+ ("time_limit_hit_before_an_edit", 2, 0, UNREAD, None, None, (TIME_LIMIT, EARLIER_DEFINITION), False, True),
+ ("measurement_failed", 2, 0, UNREAD, None, None, (0, CURRENT_DEFINITION), False, False),
+ ("measurement_failed_before_an_edit", 2, 0, UNREAD, None, None, (0, EARLIER_DEFINITION), False, True),
+ ("an_incremental_consumer_does_not_count", 2, 1, UNREAD, 3, None, None, False, False),
+ ("one_consumer_and_cheap_previews", 1, 0, CHEAP_PREVIEWS, 3, None, None, False, False),
+ ]
+ )
+ def test_proposes_a_view_that_materialized_views_compute_on_every_refresh(
+ self,
+ _name: str,
+ consumers: int,
+ incremental_consumers: int,
+ read_overrides: dict,
+ measured_days_ago: int | None,
+ edited_days_ago: int | None,
+ unfinished: tuple[int, str] | None,
+ expect_draft: bool,
+ expect_measurement_request: bool,
+ ) -> None:
+ consumer_ids = [FIRST_CONSUMER, SECOND_CONSUMER][:consumers]
+ reads = busy_reads(
+ **read_overrides,
+ measured_reads=1 if measured_days_ago else 0,
+ measured_duration_ms=TWENTY_SECONDS_MS,
+ last_measured_at=days_ago(measured_days_ago) if measured_days_ago else None,
+ )
+ candidate_context = context(
+ team_reads(
+ {view_subject(VIEW_ID): reads},
+ refreshes={
+ consumer_id: SubjectRefreshes(runs=HOURLY_RUNS, duration_ms=HOURLY_RUNS * 60_000, read_bytes=1)
+ for consumer_id in consumer_ids
+ },
+ ),
+ views=[
+ view(VIEW_ID),
+ *(
+ view(consumer_id, materializes=True, refreshes_incrementally=index < incremental_consumers)
+ for index, consumer_id in enumerate(consumer_ids)
+ ),
+ ],
+ )
+
+ with (
+ patch(f"{MATERIALIZE_MODULE}.dependent_saved_query_ids", return_value={VIEW_ID: frozenset(consumer_ids)}),
+ patch(f"{MATERIALIZE_MODULE}.saved_query_target_bounds", return_value=EVERY_BUCKET_ALLOWED),
+ patch(
+ f"{MATERIALIZE_MODULE}.effective_definition",
+ return_value=EffectiveDefinition(
+ fingerprint=CURRENT_DEFINITION,
+ changed_at=days_ago(edited_days_ago) if edited_days_ago else None,
+ ),
+ ),
+ patch(
+ f"{MATERIALIZE_MODULE}.unfinished_measurement",
+ return_value=UnfinishedMeasurement(seconds=unfinished[0], fingerprint=unfinished[1])
+ if unfinished
+ else None,
+ ),
+ patch(f"{MATERIALIZE_MODULE}.upstream_table_refs", return_value=[]),
+ patch(f"{MATERIALIZE_MODULE}.definition_database", return_value=None),
+ ):
+ result = MaterializeCandidate().evaluate(candidate_context)
+
+ assert result.measurement_requests == ((view_subject(VIEW_ID),) if expect_measurement_request else ())
+ if not expect_draft:
+ assert result.drafts == ()
+ return
+ [draft] = result.drafts
+ assert isinstance(draft.payload, MaterializePayload)
+ assert (
+ draft.payload.refresh_interval_seconds,
+ draft.payload.recomputed_by_views,
+ draft.payload.refresh_interval_basis,
+ ) == (HOUR.total_seconds(), 2, WarehouseSuggestionRefreshIntervalBasis.CONSUMER_REFRESH)
diff --git a/products/warehouse_suggestions/backend/tests/test_lineage.py b/products/warehouse_suggestions/backend/tests/test_lineage.py
new file mode 100644
index 000000000000..91561032a1a6
--- /dev/null
+++ b/products/warehouse_suggestions/backend/tests/test_lineage.py
@@ -0,0 +1,28 @@
+from uuid import uuid4
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from products.warehouse_suggestions.backend.logic.lineage import recomputing_consumers
+
+VIEW, MIDDLE, CONSUMER, BEYOND = uuid4(), uuid4(), uuid4(), uuid4()
+
+
+class TestRecomputingConsumers(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("through_a_view_that_is_not_materialized", {VIEW: [MIDDLE], MIDDLE: [CONSUMER]}, {CONSUMER}, {CONSUMER}),
+ (
+ "stops_at_the_first_materialized_view",
+ {VIEW: [CONSUMER], CONSUMER: [BEYOND]},
+ {CONSUMER, BEYOND},
+ {CONSUMER},
+ ),
+ ("survives_a_cycle", {VIEW: [MIDDLE], MIDDLE: [VIEW, CONSUMER]}, {CONSUMER}, {CONSUMER}),
+ ]
+ )
+ def test_finds_the_materialized_views_whose_refreshes_compute_it(
+ self, _name: str, dependents: dict, materialized: set, expected: set
+ ) -> None:
+ assert recomputing_consumers(VIEW, dependents, materialized) == expected
diff --git a/products/warehouse_suggestions/backend/tests/test_reads.py b/products/warehouse_suggestions/backend/tests/test_reads.py
index 51a855a07ae8..07f692feae9a 100644
--- a/products/warehouse_suggestions/backend/tests/test_reads.py
+++ b/products/warehouse_suggestions/backend/tests/test_reads.py
@@ -84,6 +84,32 @@ class TestReadTeamReads(ClickhouseTestMixin, BaseTest):
RollupRead(subject_id=view, day=TODAY - timedelta(days=40), request_id="old", user_id=3),
RollupRead(subject_id=view, day=TODAY, request_id="today", user_id=4),
RollupRead(subject_id=view, day=YESTERDAY, request_id="warmup", user_id=0, duration_ms=50),
+ RollupRead(
+ subject_id=view,
+ day=YESTERDAY,
+ request_id="consumer-refresh",
+ user_id=0,
+ lc_kind="temporal",
+ lc_feature="data_modeling",
+ duration_ms=4000,
+ ),
+ *(
+ RollupRead(
+ subject_id=view,
+ day=day,
+ request_id=f"measurement-{duration_ms}",
+ user_id=0,
+ lc_kind="temporal",
+ lc_feature="warehouse_suggestions",
+ read_alone=True,
+ duration_ms=duration_ms,
+ read_bytes=read_bytes,
+ )
+ for day, duration_ms, read_bytes in (
+ (TODAY - timedelta(days=2), 70_000, 50_000),
+ (YESTERDAY, 2000, 1000),
+ )
+ ),
RollupRead(
subject_id=view,
day=YESTERDAY,
@@ -136,7 +162,9 @@ class TestReadTeamReads(ClickhouseTestMixin, BaseTest):
assert rollup_days == RollupDays(days_with_data=3, recent_days_with_data=3)
assert (reads.readers, reads.view_readers, reads.view_reads) == (3, 2, 2)
assert (view_reads.human_requests, view_reads.human_users, view_reads.human_days) == (2, 2, 2)
- assert (view_reads.background_reads, view_reads.background_duration_ms) == (1, 50)
+ assert (view_reads.background_reads, view_reads.background_duration_ms) == (4, 76050)
+ assert (view_reads.measured_duration_ms, view_reads.measured_read_bytes) == (2000, 1000)
+ assert view_reads.last_measured_at == datetime.combine(YESTERDAY, time(hour=12), tzinfo=UTC)
refreshes = reads.refreshes[view]
assert (refreshes.runs, refreshes.duration_ms) == (2, 4090)
assert {run.workflow_id: run.duration_ms for run in reads.refresh_runs[view]} == {"run-1": 1000, "run-2": 3090}
diff --git a/products/warehouse_suggestions/backend/tests/test_temporal.py b/products/warehouse_suggestions/backend/tests/test_temporal.py
index fd7fb9ea9a74..ad5c88ed57c7 100644
--- a/products/warehouse_suggestions/backend/tests/test_temporal.py
+++ b/products/warehouse_suggestions/backend/tests/test_temporal.py
@@ -15,7 +15,9 @@ from products.warehouse_suggestions.backend.facade.enums import WarehouseSuggest
from products.warehouse_suggestions.backend.logic.candidates.registry import CANDIDATES
from products.warehouse_suggestions.backend.logic.job import TeamRunResult, TeamRunStatus
from products.warehouse_suggestions.backend.logic.reads import RollupDays
+from products.warehouse_suggestions.backend.logic.rules import RULES
from products.warehouse_suggestions.backend.logic.suggestions import upsert_suggestions
+from products.warehouse_suggestions.backend.logic.view_costs import MeasurementOutcome
from products.warehouse_suggestions.backend.models import WarehouseSuggestion
from products.warehouse_suggestions.backend.temporal import (
ACTIVITIES as WAREHOUSE_SUGGESTIONS_ACTIVITIES,
@@ -87,15 +89,22 @@ class TestTeamBatches(BaseTest):
assert batches == [[self.team.pk]]
-async def test_a_worker_runs_a_nonempty_batch_without_failing_a_team() -> None:
+async def test_a_worker_runs_a_nonempty_batch_and_measures_the_views_a_team_asked_for() -> None:
task_queue = str(uuid.uuid4())
+ unmeasured_view = uuid.uuid4()
+ measurement_requests = {1: (view_subject(unmeasured_view),), 2: ()}
with (
patch(f"{ACTIVITIES}.team_batches", return_value=[[1, 2]]),
patch(f"{ACTIVITIES}.read_rollup_days", return_value=RollupDays(days_with_data=30, recent_days_with_data=7)),
patch(
f"{ACTIVITIES}.run_team",
- side_effect=lambda team_id, **_: TeamRunResult(team_id=team_id, status=TeamRunStatus.PROCESSED),
+ side_effect=lambda team_id, **_: TeamRunResult(
+ team_id=team_id,
+ status=TeamRunStatus.PROCESSED,
+ measurement_requests=measurement_requests[team_id],
+ ),
),
+ patch(f"{ACTIVITIES}.measure_view_cost", return_value=MeasurementOutcome.MEASURED) as measure_view_cost,
):
async with await WorkflowEnvironment.start_time_skipping() as environment:
async with Worker(
@@ -113,4 +122,5 @@ async def test_a_worker_runs_a_nonempty_batch_without_failing_a_team() -> None:
result_type=BatchOutcome,
)
- assert outcome == BatchOutcome(processed=2)
+ assert outcome.processed == 2
+ measure_view_cost.assert_called_once_with(1, unmeasured_view, RULES.measurement)
diff --git a/products/warehouse_suggestions/backend/tests/test_view_costs.py b/products/warehouse_suggestions/backend/tests/test_view_costs.py
new file mode 100644
index 000000000000..7aba395ba2bc
--- /dev/null
+++ b/products/warehouse_suggestions/backend/tests/test_view_costs.py
@@ -0,0 +1,66 @@
+from posthog.test.base import BaseTest, ClickhouseTestMixin
+from unittest.mock import patch
+
+from django.core.cache import cache
+
+from parameterized import parameterized
+
+from posthog.errors import ClickHouseQueryTimeOut
+
+from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
+from products.warehouse_suggestions.backend.logic.rules import RULES
+from products.warehouse_suggestions.backend.logic.view_costs import (
+ FAILED_MEASUREMENT,
+ MeasurementOutcome,
+ measure_view_cost,
+ unfinished_measurement,
+)
+
+VIEW_COSTS_MODULE = "products.warehouse_suggestions.backend.logic.view_costs"
+
+
+class TestMeasureViewCost(ClickhouseTestMixin, BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ cache.clear()
+ self.saved_query = DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="daily_events",
+ query={"kind": "HogQLQuery", "query": "SELECT toDate(timestamp) AS day, event FROM events"},
+ columns={
+ "day": {"hogql": "DateDatabaseField", "clickhouse": "Date"},
+ "event": {"hogql": "StringDatabaseField", "clickhouse": "String"},
+ },
+ )
+
+ def test_reads_every_column_of_the_view(self) -> None:
+ outcome = measure_view_cost(self.team.pk, self.saved_query.id, RULES.measurement)
+
+ assert (outcome, unfinished_measurement(self.team.pk, self.saved_query.id)) == (
+ MeasurementOutcome.MEASURED,
+ None,
+ )
+
+ @parameterized.expand(
+ [
+ ("timeout", ClickHouseQueryTimeOut(), True, MeasurementOutcome.CAPPED, RULES.measurement.max_seconds),
+ ("other_error", ValueError("broken view"), True, MeasurementOutcome.FAILED, FAILED_MEASUREMENT),
+ ("no_stored_columns", None, False, MeasurementOutcome.FAILED, FAILED_MEASUREMENT),
+ ]
+ )
+ def test_a_measurement_that_does_not_finish_is_remembered(
+ self,
+ _name: str,
+ error: Exception | None,
+ has_columns: bool,
+ expected_outcome: MeasurementOutcome,
+ expected_cap: int,
+ ) -> None:
+ if not has_columns:
+ DataWarehouseSavedQuery.objects.filter(id=self.saved_query.id).update(columns={})
+
+ with patch(f"{VIEW_COSTS_MODULE}.execute_hogql_query", side_effect=error):
+ outcome = measure_view_cost(self.team.pk, self.saved_query.id, RULES.measurement)
+
+ unfinished = unfinished_measurement(self.team.pk, self.saved_query.id)
+ assert (outcome, unfinished.seconds if unfinished else None) == (expected_outcome, expected_cap)
diff --git a/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx b/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx
index 39c1ed8961b4..56362e845eef 100644
--- a/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx
+++ b/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx
@@ -6,7 +6,13 @@ import { LemonButton } from '@posthog/lemon-ui'
import { iconForType } from '~/layout/panel-layout/ProjectTree/defaultTree'
import type { WarehouseSuggestionApi } from '../generated/api.schemas'
-import { isMaterializePayload, materializeSaving, readSummary, savingHighlight } from '../suggestionCopy'
+import {
+ isMaterializePayload,
+ materializeSaving,
+ readSummary,
+ rebuiltSummary,
+ savingHighlight,
+} from '../suggestionCopy'
import { warehouseSuggestionsLogic } from '../warehouseSuggestionsLogic'
import { DismissSuggestionButtons } from './DismissSuggestionButtons'
import { SuggestionRow } from './SuggestionRow'
@@ -31,6 +37,8 @@ export function MaterializeSuggestionRow({
return null
}
const leaving = leavingSuggestionIds.includes(suggestion.id)
+ const rebuiltByViews = payload.recomputed_by_views ?? 0
+ const readByPeople = (suggestion.evidence.human_requests ?? 0) > 0
const blockedReason = !suggestion.can_act
? EDIT_ACCESS_REASON
: actionsInFlight[suggestion.id] || leaving
@@ -46,13 +54,16 @@ export function MaterializeSuggestionRow({
icon={iconForType('data_modeling')}
title={<SuggestionSubjectLink suggestion={suggestion} />}
meta={[
- <WhyThisSuggestion
- key="reads"
- evidence={suggestion.evidence}
- windowDays={windowDays}
- label={readSummary(suggestion.evidence)}
- />,
- ]}
+ rebuiltByViews > 0 && <span key="rebuilt">{rebuiltSummary(rebuiltByViews)}</span>,
+ (readByPeople || rebuiltByViews === 0) && (
+ <WhyThisSuggestion
+ key="reads"
+ evidence={suggestion.evidence}
+ windowDays={windowDays}
+ label={readSummary(suggestion.evidence)}
+ />
+ ),
+ ].filter(Boolean)}
highlight={savingHighlight(materializeSaving(payload))}
actions={
<>
diff --git a/products/warehouse_suggestions/frontend/generated/api.schemas.ts b/products/warehouse_suggestions/frontend/generated/api.schemas.ts
index e13e3bb7a673..14b4bc852c9d 100644
--- a/products/warehouse_suggestions/frontend/generated/api.schemas.ts
+++ b/products/warehouse_suggestions/frontend/generated/api.schemas.ts
@@ -44,6 +44,7 @@ export const WarehouseSuggestionRefreshModeEnumApi = {
* * `source_sync` - Source sync
* * `fastest_schedule` - Fastest schedule
* * `read_rate` - Read rate
+ * * `consumer_refresh` - Consumer refresh
* * `savings` - Savings
*/
export type WarehouseSuggestionRefreshIntervalBasisEnumApi =
@@ -53,6 +54,7 @@ export const WarehouseSuggestionRefreshIntervalBasisEnumApi = {
SourceSync: 'source_sync',
FastestSchedule: 'fastest_schedule',
ReadRate: 'read_rate',
+ ConsumerRefresh: 'consumer_refresh',
Savings: 'savings',
} as const
@@ -66,11 +68,12 @@ export interface WarehouseSuggestionMaterializePayloadApi {
* * `full` - Full
* * `incremental` - Incremental */
refresh_mode: WarehouseSuggestionRefreshModeEnumApi
- /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.
+ /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. consumer_refresh: the materialized views that read it refresh no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.
*
* * `source_sync` - Source sync
* * `fastest_schedule` - Fastest schedule
* * `read_rate` - Read rate
+ * * `consumer_refresh` - Consumer refresh
* * `savings` - Savings */
refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasisEnumApi | null
/** Name of the view to materialize. */
@@ -97,6 +100,8 @@ export interface WarehouseSuggestionMaterializePayloadApi {
unique_key?: string[]
/** How far before the last refresh's highest incremental_key value each refresh reads again, in seconds. The last incremental_key value is always read again. */
lookback_seconds?: number
+ /** How many materialized views compute this view again on every refresh, because it is not stored. 0 when the saving comes only from people's reads. */
+ recomputed_by_views?: number
}
export interface WarehouseSuggestionUnmaterializePayloadApi {
diff --git a/products/warehouse_suggestions/frontend/suggestionCopy.test.ts b/products/warehouse_suggestions/frontend/suggestionCopy.test.ts
index 1a3178da66a4..2a648f852dd7 100644
--- a/products/warehouse_suggestions/frontend/suggestionCopy.test.ts
+++ b/products/warehouse_suggestions/frontend/suggestionCopy.test.ts
@@ -81,6 +81,7 @@ describe('suggestionCopy', () => {
it.each<[Parameters<typeof suggestedScheduleHint>[0], string]>([
['source_sync', "sources don't sync more often"],
['read_rate', "People don't read it more often"],
+ ['consumer_refresh', "The views that read it don't refresh more often"],
['fastest_schedule', 'the most often PostHog refreshes a view'],
['savings', 'saves more than it costs'],
[null, 'saves more than it costs'],
diff --git a/products/warehouse_suggestions/frontend/suggestionCopy.ts b/products/warehouse_suggestions/frontend/suggestionCopy.ts
index 1e4b31c2be8f..0f84b7bc2dc5 100644
--- a/products/warehouse_suggestions/frontend/suggestionCopy.ts
+++ b/products/warehouse_suggestions/frontend/suggestionCopy.ts
@@ -119,6 +119,8 @@ export function suggestedScheduleHint(
return `Suggested: every ${cadence}. Its sources don't sync more often than that.`
case 'read_rate':
return `Suggested: every ${cadence}. People don't read it more often than that.`
+ case 'consumer_refresh':
+ return `Suggested: every ${cadence}. The views that read it don't refresh more often than that.`
case 'fastest_schedule':
return `Suggested: every ${cadence}, the most often PostHog refreshes a view.`
default:
@@ -135,6 +137,8 @@ export function shorterScheduleWarning(
return `Its sources sync every ${cadence} at most, so refreshing more often reads the same data again.`
case 'read_rate':
return `People don't read it more often than every ${cadence}, so extra refreshes mostly go unread.`
+ case 'consumer_refresh':
+ return `The views that read it refresh every ${cadence} at most, so refreshing more often goes unread.`
case 'fastest_schedule':
return null
default:
@@ -207,6 +211,10 @@ export function readSummary(evidence: WarehouseSuggestionEvidenceApi): string {
return `${countOf(evidence.human_requests ?? 0, 'read', 'reads')} by ${countOf(evidence.human_users ?? 0, 'person', 'people')}`
}
+export function rebuiltSummary(views: number): string {
+ return `Rebuilt on every refresh of ${countOf(views, 'view', 'views')}`
+}
+
export function daysReadLabel(evidence: WarehouseSuggestionEvidenceApi, windowDays: number): string {
return `Read on ${evidence.human_days ?? 0} of the last ${windowDays} days`
}
diff --git a/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md b/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md
index 32abda5cea97..7a3a503e484d 100644
--- a/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md
+++ b/products/warehouse_suggestions/skills/reviewing-warehouse-suggestions/SKILL.md
@@ -29,11 +29,12 @@ Its `evidence` has the read counts behind it.
Tell the user what accepting does and what it costs, using the payload fields.
Do not invent a score or a confidence level.
-- **materialize**: the view is read often enough that storing its result saves compute.
- - `saves_seconds_per_month` and `saves_bytes_per_month` are what the reads stop costing.
+- **materialize**: the view is computed often enough that storing its result saves compute.
+ - A view that is not stored is computed on every read, and again on every refresh of each materialized view that reads it. `recomputed_by_views` counts those materialized views. When it is above 0, the saving comes mostly from their refreshes, not from people's reads.
+ - `saves_seconds_per_month` and `saves_bytes_per_month` are what those computations stop costing.
- Materializing adds a recurring refresh cost, which these numbers already subtract.
- `freshness_today_seconds` and `freshness_after_seconds` say how old the data can be now and after.
- - `refresh_interval_seconds` is the proposed interval. `refresh_interval_basis` says what kept it from being shorter: `source_sync` (the sources sync no faster), `fastest_schedule` (PostHog refreshes views no more often), `read_rate` (people read the view no more often) or `savings` (a shorter interval costs more than it saves).
+ - `refresh_interval_seconds` is the proposed interval. `refresh_interval_basis` says what kept it from being shorter: `source_sync` (the sources sync no faster), `fastest_schedule` (PostHog refreshes views no more often), `read_rate` (people read the view no more often), `consumer_refresh` (the materialized views that read it refresh no more often) or `savings` (a shorter interval costs more than it saves).
- `refresh_mode` is `incremental` when each refresh can read only new rows, by `incremental_key`, and update rows by `unique_key`. Otherwise it is `full`, and each refresh rebuilds the whole table.
- **unmaterialize**: this materialized view is read so rarely that refreshing it costs more than computing it on each read.
- `refresh_seconds_per_month` and `refresh_bytes_per_month` are what its refreshes cost now.
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index d920f39d2b5f..a2b6ef01eec1 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -83020,6 +83020,7 @@ export namespace Schemas {
* * `source_sync` - Source sync
* * `fastest_schedule` - Fastest schedule
* * `read_rate` - Read rate
+ * * `consumer_refresh` - Consumer refresh
* * `savings` - Savings
*/
export type WarehouseSuggestionRefreshIntervalBasisEnum = typeof WarehouseSuggestionRefreshIntervalBasisEnum[keyof typeof WarehouseSuggestionRefreshIntervalBasisEnum];
@@ -83029,6 +83030,7 @@ export namespace Schemas {
SourceSync: 'source_sync',
FastestSchedule: 'fastest_schedule',
ReadRate: 'read_rate',
+ ConsumerRefresh: 'consumer_refresh',
Savings: 'savings',
} as const;
@@ -83042,11 +83044,12 @@ export namespace Schemas {
* * `full` - Full
* * `incremental` - Incremental */
refresh_mode: WarehouseSuggestionRefreshModeEnum;
- /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.
+ /** Why the interval is not shorter. source_sync: the view's sources sync no faster. fastest_schedule: PostHog refreshes views no more often. read_rate: people read the view no more often. consumer_refresh: the materialized views that read it refresh no more often. savings: a shorter interval would cost more than it saves. Null on suggestions made before this was recorded.
*
* * `source_sync` - Source sync
* * `fastest_schedule` - Fastest schedule
* * `read_rate` - Read rate
+ * * `consumer_refresh` - Consumer refresh
* * `savings` - Savings */
refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasisEnum | null;
/** Name of the view to materialize. */
@@ -83073,6 +83076,8 @@ export namespace Schemas {
unique_key?: string[];
/** How far before the last refresh's highest incremental_key value each refresh reads again, in seconds. The last incremental_key value is always read again. */
lookback_seconds?: number;
+ /** How many materialized views compute this view again on every refresh, because it is not stored. 0 when the saving comes only from people's reads. */
+ recomputed_by_views?: number;
}
export interface WarehouseSuggestionUnmaterializePayload {