Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-suggestions): lead each saving with what earned it (#125756)

Thiago Salvatore516f4c914515
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index eddd39b6ce3e..4541b217141b 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -17503,45 +17503,45 @@ snapshots:
warehouse-suggestions-suggested-certifications--review-dialog-narrow--light:
hash: v1.k794b7964.b50eac0dbd82aa276d178cebb9041a89ba689f376f3b8509b910aa3a291a4441.6LJGfUarId3i8ZJYqoBfGZ1zaK_btz8bo1q6t4RSs38
warehouse-suggestions-suggested-models-strip--banner--dark:
- hash: v1.k794b7964.95ba4716f680e32e7497d3f5d8dbcdca2f9da83d7e08e5b2bb8911a7ab2c39fb.ackP53GYozYXF74XgG1MVnH6degCJR3pkvrXGKJJ3hw
+ hash: v1.k794b7964.f4b1753fb8f7f72bc726754edc2af9890d58bad769aef29e0535aaaa9862a0cb.DJOIQ1B3uve853HT-qkIbooHSJMhcVHE1F0DGqrTQ0g
warehouse-suggestions-suggested-models-strip--banner--light:
- hash: v1.k794b7964.e5c0612cb28d10889c2e12d45e301179b8486fd9c4955bc7c263a0bbeb3e48d1.HaP6FZCwy8IjEBBE3PN0v965whOBqLoHFu2ydHAP42w
+ hash: v1.k794b7964.067b7d595dfb3c45861e60e1c479bcf3b1093e748f996ae628b60d8e3582449a.e4cUdU_PGjeC9mqyBHwozZ8UhuFadtIVbU2HV2_d9tA
warehouse-suggestions-suggested-models-strip--banner-narrow--dark:
- hash: v1.k794b7964.a9f0234e6bbdb3fddad0a01f6412e0e04b441b242b584398f294310f11f7fb90.t0yZHTlHGZRUmTmMvB222jaOopi7-Qp33bjpXHvVbr0
+ hash: v1.k794b7964.6d5603167d875c5dd2e3a48a329e18d5ac951d63471d833ed1384d1c18761212.4UrtyPwOVvgs4Kl-9zJsXXgzYwJs8d1IIanHEcjCMes
warehouse-suggestions-suggested-models-strip--banner-narrow--light:
- hash: v1.k794b7964.4853cd549edce121e6b9f987c5bf96dfac76926581745f457f5421afdf0d0bf1.0z4Cvnvp5G-qwQBMck2dkiQSMmEOF3JMwp5dDT5Dk38
+ hash: v1.k794b7964.4e1894a911c620104b0cb8a776ccaab6b31818a2733caa0a6cb5ea414e5c009c.qOnUdghMNhGn8b9VTGLfRwpF5XA5e6TKTfAv97cFjeo
warehouse-suggestions-suggested-models-strip--load-failed--dark:
hash: v1.k794b7964.d93ab3575b7bebf7f869bc5b41c0a8e0581679f22bd0251c31f031b0f43ee7c0.iXXukenkQJA0ayAlRoqei_RQx1XW-b9MLaTjdhxfkoM
warehouse-suggestions-suggested-models-strip--load-failed--light:
hash: v1.k794b7964.a5b60b08e7b0a541c14bae9a0a40118fced381d13bfdd40b8e93342959f24f19.LwTgelFpFnRFgNgu3ueAFF9dCIqGgJGMgoyv53ZJe9g
warehouse-suggestions-suggested-models-strip--materialize-dialog-incremental-proposal--dark:
- hash: v1.k794b7964.032b237f6b064b97c2967bd3371fc5976b56d2bc53bc7c1477ccb179d4328bba.VLngPFGD7X4zR-A3lBUzSXTlEFyxiyJSRJrSW9nASVQ
+ hash: v1.k794b7964.e64669a1ff38b486671b051441cadf12397cf2acea8e7b7e857793d91862cd0d.Lpngke-C7_OjyFTAhpH6sRT06b-_oB2llFgUDOBzwDQ
warehouse-suggestions-suggested-models-strip--materialize-dialog-incremental-proposal--light:
- hash: v1.k794b7964.615fefa2b72edb8e3ac52f8a40b643d70e2266cf4087ec2e530ea416a663737c.a3kwJ5OMMvvy16QIvO_03BUCME2Ludg6KRs9zLcMsjw
+ hash: v1.k794b7964.a6beb21aecd3fad1e759657f6de3fe5cd03a36a01701eb41920a56fed2a6b69b.T6Pws4xC1tCnqkgGk6GEiEhpawBNkQ5O0JZSa863ERo
warehouse-suggestions-suggested-models-strip--materialize-dialog-longer-schedule-unknown-sources--dark:
- hash: v1.k794b7964.837f31cbe827ec69c755f8423ec7d5e42176a1805d981b43752c770785121525.2iRbK4dl5sZVs6YlAWvlp5yksawu-ePXJEJkqefBOBI
+ hash: v1.k794b7964.3bc1c3deb6c910316863b9d6f1a5c6c653cf4b49ae8d775efa772a457e249292.ySDxfOrQXte19z-xOkD0-DjUF8X9d7c2SF9QKJTDo30
warehouse-suggestions-suggested-models-strip--materialize-dialog-longer-schedule-unknown-sources--light:
- hash: v1.k794b7964.14635082a49901aafdf5cae1f184665676ca385cc9c3b4b5288279d158aa5151.k1NZ1ZVAxd9pGr0zl8OjMJcubGjeVcEanuNve-CnsiI
+ hash: v1.k794b7964.67b7faf54be86b63f9ccf7bfba080561255b5f699eab8158a60d2ffa56d908d1.Ry5JM7XpQfk6FfxR5l7O2dEADcdfPyo3weg4dB_aeKs
warehouse-suggestions-suggested-models-strip--not-eligible--dark:
hash: v1.k794b7964.b646d37c91e92fd3d0053e1407559117a50c06c45f8ff7f4bcaaed18bab1cfef.1vguAxzgXrCyuyN1v2jRBSTSGDrmsu62f3nO60GFvGc
warehouse-suggestions-suggested-models-strip--not-eligible--light:
hash: v1.k794b7964.a66ac5be916b459b6f8b15621e3bdeb12226d34c1e24feb15900940652fc238b.gzxwnKSQkZQfg5wdIMriKJO-aaxIteiXVzfSx1R7w8c
warehouse-suggestions-suggested-models-strip--review-dialog--dark:
- hash: v1.k794b7964.6704636139e679bff06ae8ac293a11a79512d2ebb1b40ba860c5b92cf68914f9.kV00kEDOPcXei1uIUD98ZpiGTl7yWEKLc6gM-cB68QQ
+ hash: v1.k794b7964.3ce6732f7e7e53eed4c1b97ae82d929f55597946c16dc485a44033e87c0d812e.jXmFDVw5SUPT6kShyBsznWyAfdlqs0kK1RFF8SJFxsY
warehouse-suggestions-suggested-models-strip--review-dialog--light:
- hash: v1.k794b7964.ee48017846790bc155b3d56bf09deb212250a7b8c0cc109f80b17c2129ee9678.XXIsgSRudiZQJhR5z6up89pCNSrNxZ8cN4Idf6Lc9gk
+ hash: v1.k794b7964.a3626ebfb53ca9f53ee5e6437d50163c686a5fcc56a2fa3a7223751d110462c9.hVKPaFbueFpouk0eaqw0FHLM_tXQFyAomVQCr9vxYfs
warehouse-suggestions-suggested-models-strip--review-dialog-many-suggestions--dark:
- hash: v1.k794b7964.1fe85a2835e1f5b317fc8c9158864742c8cc75923b0dbd46a511a60353f4c8c2.U8fMH_bEQlPHWoc33EeE3R8PIyeVhkymf-tNrBqrD3U
+ hash: v1.k794b7964.825b125bcc53a5c7f1141c6b51fb2bd4f0d5afdf70858a557571f30d619241d8.q0jvTBZUDDrMwbWchF-gUZ1_QRtCdguQA0xWIZvIjiU
warehouse-suggestions-suggested-models-strip--review-dialog-many-suggestions--light:
- hash: v1.k794b7964.49dc6fecf31a0af324cd77fab6201ed3b56d9c0924eef3fe0152568efe000a67.naBylAMmZJ8rGTSgezEL-SsV-CDkWfPDTV16Cbo4ed0
+ hash: v1.k794b7964.91eba9d19e65353cfec9b2f85b7cf13891358f03c2eae99e4b3e7c952b9439c2.gkvwU-fA9qi758_y_I2VrO_MK7xzAcMHG94raaWffV8
warehouse-suggestions-suggested-models-strip--review-dialog-viewer-without-edit-access--dark:
- hash: v1.k794b7964.fd6be2b86f1b8d3c97f42d616f18bd03cd847c1aab8b2b1705c28432eb9f82c4.YjRSp1tHz_wptuaPolsSoQ5UA2xJx9KTlx9Uts3f6R4
+ hash: v1.k794b7964.ae462ddadc20cc15be3a9bdbae777bafd62ee370fe89a221e63981947b7e5db8.FyOX4NtW10hzt0fMbxk5YuMvtlUdmMGJCK7q7nM3-xE
warehouse-suggestions-suggested-models-strip--review-dialog-viewer-without-edit-access--light:
- hash: v1.k794b7964.e28785f4439ef22c26f953323b5e859d0822054f6b92396eb237c535b9f0a41e.Lj4zRNVtLh5Nolx2M5A8OSnyuUG_fv2ghcjzkd2mGjg
+ hash: v1.k794b7964.384d979d79f514d57c0437cb8537aa33c3cb9605bae6b4232bfb6a0d0780ab1f.mpr7tkufxhbHtPc7hVRiYubOViBJqijmVf_V0Fud2vU
warehouse-suggestions-suggested-models-strip--stop-materializing-dialog--dark:
- hash: v1.k794b7964.f1bbacf85ccb9c90e8980f0fb2b76e9a88dc44ae6c0fc7b438ae175166fbc107.clj4aF0nN4fRaxCxUKGQ2D9Heh3ztyfzBq_yJYD3B6o
+ hash: v1.k794b7964.3de8918a7d4549c7b94f1880811300aa0be9ad3b43f0ca0ddbf7ce4263cbff5f.JEGq_DJuFHfCX_EXG0nFvX009s0vOUZj8_mSnZTXKEE
warehouse-suggestions-suggested-models-strip--stop-materializing-dialog--light:
- hash: v1.k794b7964.24a5826c28aedc9559806790ce28842b7c878d79efd405ffdefe5e88540b1b76.sfIdeW-Q3n9Zny2O-_OVvnRewQjqNytyAkzYljzaEoU
+ hash: v1.k794b7964.2d92023ad6d3dc2226605434a4ae789ddae657f3f8a5750a6279a55780ad4139.DnwVKCy6Pmp3hY9LRaIzCtirVwlhWxThIosF4FZV35g
warehouse-suggestions-suggested-models-strip--warming-up--dark:
hash: v1.k794b7964.860407091149fedd6c4e056cda79d92213e59dd9e00886e0cedee320fc267c48.x5QcsRUSRT66uOc237oq_n4D-eQWpY2GzsTRTC-SL6w
warehouse-suggestions-suggested-models-strip--warming-up--light:
diff --git a/products/warehouse_suggestions/backend/facade/contracts.py b/products/warehouse_suggestions/backend/facade/contracts.py
index deb74073612d..753c3137a385 100644
--- a/products/warehouse_suggestions/backend/facade/contracts.py
+++ b/products/warehouse_suggestions/backend/facade/contracts.py
@@ -10,6 +10,7 @@ from .enums import (
WarehouseSuggestionAssetOutcome,
WarehouseSuggestionDismissalReason,
WarehouseSuggestionKind,
+ WarehouseSuggestionMainSaving,
WarehouseSuggestionRefreshIntervalBasis,
WarehouseSuggestionRefreshMode,
WarehouseSuggestionStatus,
@@ -50,8 +51,8 @@ class DeprecatePayload:
@frozen
-class MaterializePayload:
- """The stored materialize payload, with the warehouse table id of each source."""
+class MaterializePayloadFields:
+ """The materialize fields the stored payload and the payload a caller sees share."""
subject_name: str
refresh_interval_seconds: int
@@ -59,14 +60,24 @@ class MaterializePayload:
saves_bytes_per_month: float
freshness_today_seconds: int | None
freshness_after_seconds: int
- live_sources: tuple[SourceRef, ...]
- unknown_sources: tuple[SourceRef, ...]
refresh_mode: WarehouseSuggestionRefreshMode = WarehouseSuggestionRefreshMode.FULL
incremental_key: str | None = None
unique_key: tuple[str, ...] = ()
lookback_seconds: int = 0
refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasis | None = None
recomputed_by_views: int = 0
+ waiting_seconds_saved_per_month: float | None = None
+ background_seconds_saved_per_month: float | None = None
+ main_saving: WarehouseSuggestionMainSaving | None = None
+ read_seconds: float | None = None
+
+
+@frozen
+class MaterializePayload(MaterializePayloadFields):
+ """The stored materialize payload, with the warehouse table id of each source."""
+
+ live_sources: tuple[SourceRef, ...]
+ unknown_sources: tuple[SourceRef, ...]
@frozen
@@ -90,23 +101,11 @@ class VisibleSources:
@frozen
-class MaterializeSuggestionPayload:
+class MaterializeSuggestionPayload(MaterializePayloadFields):
"""The materialize payload a caller sees: the source names they can read, and a count of the others."""
- subject_name: str
- refresh_interval_seconds: int
- saves_seconds_per_month: float
- saves_bytes_per_month: float
- freshness_today_seconds: int | None
- freshness_after_seconds: int
live_sources: VisibleSources
unknown_sources: VisibleSources
- refresh_mode: WarehouseSuggestionRefreshMode = WarehouseSuggestionRefreshMode.FULL
- incremental_key: str | None = None
- unique_key: tuple[str, ...] = ()
- lookback_seconds: int = 0
- refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasis | None = None
- recomputed_by_views: int = 0
SuggestionPayloadView = CertifyPayload | DeprecatePayload | MaterializeSuggestionPayload | UnmaterializePayload
diff --git a/products/warehouse_suggestions/backend/facade/enums.py b/products/warehouse_suggestions/backend/facade/enums.py
index ad05c93ac071..2d46298386dd 100644
--- a/products/warehouse_suggestions/backend/facade/enums.py
+++ b/products/warehouse_suggestions/backend/facade/enums.py
@@ -40,6 +40,12 @@ class WarehouseSuggestionRefreshMode(LabeledStrEnum):
INCREMENTAL = "incremental"
+class WarehouseSuggestionMainSaving(LabeledStrEnum):
+ WAITING = "waiting"
+ BACKGROUND = "background"
+ SCANNED = "scanned"
+
+
class WarehouseSuggestionRefreshIntervalBasis(LabeledStrEnum):
SOURCE_SYNC = ("source_sync", "Source sync")
FASTEST_SCHEDULE = ("fastest_schedule", "Fastest schedule")
diff --git a/products/warehouse_suggestions/backend/logic/candidates/materialize.py b/products/warehouse_suggestions/backend/logic/candidates/materialize.py
index 6298253bc8e1..88cee32d3586 100644
--- a/products/warehouse_suggestions/backend/logic/candidates/materialize.py
+++ b/products/warehouse_suggestions/backend/logic/candidates/materialize.py
@@ -26,6 +26,7 @@ from ...facade.contracts import MaterializePayload, SourceRef
from ...facade.enums import (
WarehouseSuggestionAssetOutcome,
WarehouseSuggestionKind,
+ WarehouseSuggestionMainSaving,
WarehouseSuggestionRefreshIntervalBasis,
WarehouseSuggestionRefreshMode,
WarehouseSuggestionSubjectKind,
@@ -63,6 +64,9 @@ class SavingsEstimate:
read_seconds: float
read_bytes: float
seconds_saved: float
+ waiting_seconds_saved: float
+ background_seconds_saved: float
+ main_saving: WarehouseSuggestionMainSaving
bytes_saved: float
clears_a_floor: bool
score: float
@@ -263,31 +267,13 @@ class MaterializeCandidate(Candidate):
)
if estimate is None:
return "savings stay under the floors at every allowed refresh interval"
- sources = _classify_sources(context, saved_query.id, frequency)
- freshness_today = int(frequency.max_data_age.total_seconds()) or None
- interval_seconds = int(estimate.interval.total_seconds())
- payload = MaterializePayload(
- subject_name=saved_query.name,
- refresh_interval_seconds=interval_seconds,
- saves_seconds_per_month=estimate.seconds_saved,
- saves_bytes_per_month=estimate.bytes_saved,
- freshness_today_seconds=freshness_today,
- freshness_after_seconds=interval_seconds + (freshness_today or 0),
- live_sources=sources.live,
- unknown_sources=sources.unknown,
- refresh_interval_basis=interval_basis(
- estimate.interval,
- frequency.bounds.allowed,
- intervals,
- limited_by_sources=frequency.bounds.floor is not None,
- ),
- )
- return _Proposal(
- payload=_with_refresh_mode(
- payload, incremental.config_for(saved_query), context.rules.materialize.incremental_lookback_seconds
- ),
- estimate=estimate,
+ basis = interval_basis(
+ estimate.interval,
+ frequency.bounds.allowed,
+ intervals,
+ limited_by_sources=frequency.bounds.floor is not None,
)
+ return _proposal(context, saved_query, reads, frequency, estimate, incremental, basis=basis)
def _propose_recomputed(
self,
@@ -324,27 +310,16 @@ class MaterializeCandidate(Candidate):
estimate = recomputation_savings(context, reads, recomputations, interval, cost)
if not estimate.clears_a_floor:
return "savings stay under the floors at the interval the views that read it need"
- sources = _classify_sources(context, saved_query.id, frequency)
- freshness_today = int(frequency.max_data_age.total_seconds()) or None
- interval_seconds = int(interval.total_seconds())
- payload = MaterializePayload(
- subject_name=saved_query.name,
- refresh_interval_seconds=interval_seconds,
- saves_seconds_per_month=estimate.seconds_saved,
- saves_bytes_per_month=estimate.bytes_saved,
- freshness_today_seconds=freshness_today,
- freshness_after_seconds=interval_seconds + (freshness_today or 0),
- live_sources=sources.live,
- unknown_sources=sources.unknown,
- refresh_interval_basis=WarehouseSuggestionRefreshIntervalBasis.CONSUMER_REFRESH,
+ return _proposal(
+ context,
+ saved_query,
+ reads,
+ frequency,
+ estimate,
+ incremental,
+ basis=WarehouseSuggestionRefreshIntervalBasis.CONSUMER_REFRESH,
recomputed_by_views=recomputations.consumers,
)
- return _Proposal(
- payload=_with_refresh_mode(
- payload, incremental.config_for(saved_query), context.rules.materialize.incremental_lookback_seconds
- ),
- estimate=estimate,
- )
def _read_floor_rejection(self, context: CandidateContext, reads: SubjectReads) -> str | None:
rules = context.rules.materialize
@@ -386,6 +361,44 @@ def interval_basis(
return WarehouseSuggestionRefreshIntervalBasis.SAVINGS
+def _proposal(
+ context: CandidateContext,
+ saved_query: SavedQueryDefinition,
+ reads: SubjectReads | None,
+ frequency: SavedQueryFrequencyBounds,
+ estimate: SavingsEstimate,
+ incremental: _IncrementalProposer,
+ *,
+ basis: WarehouseSuggestionRefreshIntervalBasis,
+ recomputed_by_views: int = 0,
+) -> _Proposal:
+ sources = _classify_sources(context, saved_query.id, frequency)
+ freshness_today = int(frequency.max_data_age.total_seconds()) or None
+ interval_seconds = int(estimate.interval.total_seconds())
+ payload = MaterializePayload(
+ subject_name=saved_query.name,
+ refresh_interval_seconds=interval_seconds,
+ saves_seconds_per_month=estimate.seconds_saved,
+ saves_bytes_per_month=estimate.bytes_saved,
+ waiting_seconds_saved_per_month=estimate.waiting_seconds_saved,
+ background_seconds_saved_per_month=estimate.background_seconds_saved,
+ main_saving=estimate.main_saving,
+ read_seconds=_read_seconds(reads),
+ freshness_today_seconds=freshness_today,
+ freshness_after_seconds=interval_seconds + (freshness_today or 0),
+ live_sources=sources.live,
+ unknown_sources=sources.unknown,
+ refresh_interval_basis=basis,
+ recomputed_by_views=recomputed_by_views,
+ )
+ return _Proposal(
+ payload=_with_refresh_mode(
+ payload, incremental.config_for(saved_query), context.rules.materialize.incremental_lookback_seconds
+ ),
+ estimate=estimate,
+ )
+
+
def _with_refresh_mode(
payload: MaterializePayload, config: IncrementalConfig | None, lookback_seconds: int
) -> MaterializePayload:
@@ -421,9 +434,9 @@ def estimate_savings(
context.rules.materialize,
interval,
cost_without=read_costs.observed,
- saved=QueryCost(
- seconds=read_costs.saved.seconds - refreshes_per_month * refresh.seconds,
- read_bytes=read_costs.saved.read_bytes - refreshes_per_month * refresh.read_bytes,
+ waiting_saved=read_costs.saved,
+ background_saved=QueryCost(
+ seconds=-refreshes_per_month * refresh.seconds, read_bytes=-refreshes_per_month * refresh.read_bytes
),
)
@@ -497,16 +510,23 @@ def recomputation_savings(
seconds=runs * cost.seconds + read_costs.observed.seconds,
read_bytes=runs * cost.read_bytes + read_costs.observed.read_bytes,
),
- saved=QueryCost(
- seconds=runs_saved * cost.seconds + read_costs.saved.seconds,
- read_bytes=runs_saved * cost.read_bytes + read_costs.saved.read_bytes,
- ),
+ waiting_saved=read_costs.saved,
+ background_saved=QueryCost(seconds=runs_saved * cost.seconds, read_bytes=runs_saved * cost.read_bytes),
)
def _estimate(
- rules: MaterializeRules, interval: timedelta, *, cost_without: QueryCost, saved: QueryCost
+ rules: MaterializeRules,
+ interval: timedelta,
+ *,
+ cost_without: QueryCost,
+ waiting_saved: QueryCost,
+ background_saved: QueryCost,
) -> SavingsEstimate:
+ saved = QueryCost(
+ seconds=waiting_saved.seconds + background_saved.seconds,
+ read_bytes=waiting_saved.read_bytes + background_saved.read_bytes,
+ )
clears_time = (
saved.seconds >= rules.min_seconds_saved and saved.seconds >= rules.min_saved_share * cost_without.seconds
)
@@ -514,17 +534,37 @@ def _estimate(
saved.read_bytes >= rules.min_bytes_saved
and saved.read_bytes >= rules.min_saved_share * cost_without.read_bytes
)
+ time_ratio = saved.seconds / rules.min_seconds_saved
+ bytes_ratio = saved.read_bytes / rules.min_bytes_saved
+ bytes_lead = clears_bytes and (not clears_time or bytes_ratio > time_ratio)
return SavingsEstimate(
interval=interval,
read_seconds=cost_without.seconds,
read_bytes=cost_without.read_bytes,
seconds_saved=saved.seconds,
+ waiting_seconds_saved=waiting_saved.seconds,
+ background_seconds_saved=background_saved.seconds,
+ main_saving=_main_saving(bytes_lead, waiting_saved.seconds, background_saved.seconds),
bytes_saved=saved.read_bytes,
clears_a_floor=clears_time or clears_bytes,
- score=max(saved.seconds / rules.min_seconds_saved, saved.read_bytes / rules.min_bytes_saved),
+ score=max(time_ratio, bytes_ratio),
)
+def _main_saving(bytes_lead: bool, waiting_seconds: float, background_seconds: float) -> WarehouseSuggestionMainSaving:
+ if bytes_lead:
+ return WarehouseSuggestionMainSaving.SCANNED
+ if waiting_seconds >= background_seconds:
+ return WarehouseSuggestionMainSaving.WAITING
+ return WarehouseSuggestionMainSaving.BACKGROUND
+
+
+def _read_seconds(reads: SubjectReads | None) -> float | None:
+ if reads is None or reads.alone_reads == 0:
+ return None
+ return reads.alone_duration_ms_median / MILLISECONDS_PER_SECOND
+
+
def _most_recomputed(unmeasured: list[tuple[Subject, Recomputations]], limit: int) -> tuple[Subject, ...]:
ranked = sorted(unmeasured, key=lambda pair: pair[1].consumer_runs_per_month, reverse=True)
return tuple(subject for subject, _ in ranked[:limit])
diff --git a/products/warehouse_suggestions/backend/logic/payloads.py b/products/warehouse_suggestions/backend/logic/payloads.py
index 6b2412dee8c6..61a79c563fd0 100644
--- a/products/warehouse_suggestions/backend/logic/payloads.py
+++ b/products/warehouse_suggestions/backend/logic/payloads.py
@@ -19,6 +19,7 @@ from ..facade.contracts import (
)
from ..facade.enums import (
WarehouseSuggestionKind,
+ WarehouseSuggestionMainSaving,
WarehouseSuggestionRefreshIntervalBasis,
WarehouseSuggestionRefreshMode,
)
@@ -88,8 +89,10 @@ def _source_ref(source: Mapping[str, Any]) -> SourceRef:
def _refresh_fields(data: Mapping[str, Any]) -> dict[str, Any]:
basis = data.get("refresh_interval_basis")
+ main_saving = data.get("main_saving")
return {
"refresh_mode": WarehouseSuggestionRefreshMode(data.get("refresh_mode", WarehouseSuggestionRefreshMode.FULL)),
"unique_key": tuple(data.get("unique_key", ())),
"refresh_interval_basis": WarehouseSuggestionRefreshIntervalBasis(basis) if basis else None,
+ "main_saving": WarehouseSuggestionMainSaving(main_saving) if main_saving else None,
}
diff --git a/products/warehouse_suggestions/backend/presentation/serializers.py b/products/warehouse_suggestions/backend/presentation/serializers.py
index 46d64e17b9a0..6ec608c90dd2 100644
--- a/products/warehouse_suggestions/backend/presentation/serializers.py
+++ b/products/warehouse_suggestions/backend/presentation/serializers.py
@@ -25,6 +25,7 @@ from ..facade.enums import (
WarehouseSuggestionAssetOutcome,
WarehouseSuggestionDismissalReason,
WarehouseSuggestionKind,
+ WarehouseSuggestionMainSaving,
WarehouseSuggestionRefreshIntervalBasis,
WarehouseSuggestionRefreshMode,
WarehouseSuggestionStatus,
@@ -105,6 +106,13 @@ class MaterializePayloadSerializer(DataclassSerializer):
"savings: a shorter interval would cost more than it saves. Null on suggestions made before this was "
"recorded.",
)
+ main_saving = serializers.ChoiceField(
+ choices=WarehouseSuggestionMainSaving.choices,
+ allow_null=True,
+ help_text="The saving that earned the suggestion. waiting: people, agents and API callers wait less on "
+ "reads. background: refreshes do less work. scanned: reads scan fewer bytes. Null on suggestions made before "
+ "this was recorded.",
+ )
class Meta:
dataclass = MaterializeSuggestionPayload
@@ -134,6 +142,19 @@ class MaterializePayloadSerializer(DataclassSerializer):
"help_text": "How many materialized views compute this view again on every refresh, because it "
"is not stored. 0 when the saving comes only from people's reads."
},
+ "waiting_seconds_saved_per_month": {
+ "help_text": "Part of saves_seconds_per_month that people, agents and API callers spend waiting on "
+ "reads of this view, in seconds. Null on suggestions made before this was recorded."
+ },
+ "background_seconds_saved_per_month": {
+ "help_text": "Part of saves_seconds_per_month that background refreshes spend, in seconds. Negative "
+ "when refreshing the stored copy costs more background time than it saves. Null on suggestions made "
+ "before this was recorded."
+ },
+ "read_seconds": {
+ "help_text": "How long one read of the view takes today, in seconds. Null when no read was "
+ "measured on its own."
+ },
}
diff --git a/products/warehouse_suggestions/backend/tests/test_candidates.py b/products/warehouse_suggestions/backend/tests/test_candidates.py
index 4bd79a7dab78..6b660438725e 100644
--- a/products/warehouse_suggestions/backend/tests/test_candidates.py
+++ b/products/warehouse_suggestions/backend/tests/test_candidates.py
@@ -24,6 +24,7 @@ from products.warehouse_suggestions.backend.facade.contracts import (
from products.warehouse_suggestions.backend.facade.enums import (
WarehouseSuggestionAssetOutcome,
WarehouseSuggestionKind,
+ WarehouseSuggestionMainSaving,
WarehouseSuggestionRefreshIntervalBasis,
WarehouseSuggestionRefreshMode,
)
@@ -217,6 +218,33 @@ class TestMaterializeRules(SimpleTestCase):
assert estimate.clears_a_floor is expect_clear
+ @parameterized.expand(
+ [
+ ("time_earns_it", 1, WarehouseSuggestionMainSaving.WAITING),
+ ("bytes_earn_it", 2 * 1024**4, WarehouseSuggestionMainSaving.SCANNED),
+ ]
+ )
+ def test_reads_save_waiting_time_and_refreshes_cost_background_time(
+ self, _name: str, bytes_per_read: int, main_saving: WarehouseSuggestionMainSaving
+ ) -> None:
+ reads = busy_reads(
+ human_reads=90,
+ human_duration_ms=90 * 10_000,
+ alone_duration_ms_median=10_000.0,
+ human_read_bytes=90 * bytes_per_read,
+ alone_read_bytes_median=float(bytes_per_read),
+ )
+ ctx = context(team_reads({}, refreshes={VIEW_ID: SubjectRefreshes(runs=3, duration_ms=30_000, read_bytes=3)}))
+
+ estimate = estimate_savings(ctx, reads, DAY, refresh_cost(ctx, VIEW_ID, reads))
+
+ assert (
+ estimate.waiting_seconds_saved,
+ estimate.background_seconds_saved,
+ estimate.seconds_saved,
+ estimate.main_saving,
+ ) == (900, -300, 600, main_saving)
+
@parameterized.expand(
[
(
@@ -755,4 +783,14 @@ class TestMaterializeRecomputedUpstream(SimpleTestCase):
draft.payload.refresh_interval_seconds,
draft.payload.recomputed_by_views,
draft.payload.refresh_interval_basis,
- ) == (HOUR.total_seconds(), 2, WarehouseSuggestionRefreshIntervalBasis.CONSUMER_REFRESH)
+ draft.payload.waiting_seconds_saved_per_month,
+ draft.payload.background_seconds_saved_per_month,
+ draft.payload.main_saving,
+ ) == (
+ HOUR.total_seconds(),
+ 2,
+ WarehouseSuggestionRefreshIntervalBasis.CONSUMER_REFRESH,
+ 0,
+ draft.payload.saves_seconds_per_month,
+ WarehouseSuggestionMainSaving.BACKGROUND,
+ )
diff --git a/products/warehouse_suggestions/frontend/components/MaterializeSuggestionModal.tsx b/products/warehouse_suggestions/frontend/components/MaterializeSuggestionModal.tsx
index bfdff84c3c42..89d514f4b20a 100644
--- a/products/warehouse_suggestions/frontend/components/MaterializeSuggestionModal.tsx
+++ b/products/warehouse_suggestions/frontend/components/MaterializeSuggestionModal.tsx
@@ -12,9 +12,8 @@ import { INTERVAL_SECONDS, intervalForSeconds } from '../refreshIntervals'
import {
dataAgeLabel,
isMaterializePayload,
- materializeSaving,
+ materializeSavingLines,
refreshModeLabel,
- savingSentence,
shorterScheduleWarning,
suggestedScheduleHint,
todayAgeLabel,
@@ -45,6 +44,7 @@ export function MaterializeSuggestionModal(): JSX.Element | null {
const suggestedInterval = intervalForSeconds(payload.refresh_interval_seconds)
const suggestedCadence = suggestedInterval ? CADENCE_LABELS[suggestedInterval] : null
const pickedSeconds = materializeInterval ? INTERVAL_SECONDS[materializeInterval] : payload.refresh_interval_seconds
+ const [estimateHeadline, ...estimateDetails] = materializeSavingLines(payload)
const shorterWarning =
materializeIntervalShorterThanSuggested && suggestedCadence
? shorterScheduleWarning(payload.refresh_interval_basis, suggestedCadence)
@@ -116,7 +116,12 @@ export function MaterializeSuggestionModal(): JSX.Element | null {
<span className="text-xs font-semibold text-secondary">
Estimated at the suggested settings
</span>
- <span className="text-sm">{savingSentence(materializeSaving(payload))}</span>
+ <span className="text-sm">{estimateHeadline}</span>
+ {estimateDetails.map((line) => (
+ <span key={line} className="text-xs text-secondary">
+ {line}
+ </span>
+ ))}
</div>
</div>
{shorterWarning && <LemonBanner type="warning">{shorterWarning}</LemonBanner>}
diff --git a/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx b/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx
index 56362e845eef..11464938de15 100644
--- a/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx
+++ b/products/warehouse_suggestions/frontend/components/MaterializeSuggestionRow.tsx
@@ -6,13 +6,7 @@ import { LemonButton } from '@posthog/lemon-ui'
import { iconForType } from '~/layout/panel-layout/ProjectTree/defaultTree'
import type { WarehouseSuggestionApi } from '../generated/api.schemas'
-import {
- isMaterializePayload,
- materializeSaving,
- readSummary,
- rebuiltSummary,
- savingHighlight,
-} from '../suggestionCopy'
+import { isMaterializePayload, materializeHighlight, readSummary, rebuiltSummary } from '../suggestionCopy'
import { warehouseSuggestionsLogic } from '../warehouseSuggestionsLogic'
import { DismissSuggestionButtons } from './DismissSuggestionButtons'
import { SuggestionRow } from './SuggestionRow'
@@ -64,7 +58,7 @@ export function MaterializeSuggestionRow({
/>
),
].filter(Boolean)}
- highlight={savingHighlight(materializeSaving(payload))}
+ highlight={materializeHighlight(payload)}
actions={
<>
<LemonButton
diff --git a/products/warehouse_suggestions/frontend/components/SavingsBannerMessage.tsx b/products/warehouse_suggestions/frontend/components/SavingsBannerMessage.tsx
index 4d6504b68dc5..13cf3d64536d 100644
--- a/products/warehouse_suggestions/frontend/components/SavingsBannerMessage.tsx
+++ b/products/warehouse_suggestions/frontend/components/SavingsBannerMessage.tsx
@@ -1,33 +1,48 @@
-import { SavingsSummary, countOf, namedSaving } from '../suggestionCopy'
+import { SavingsSummary, bannerSavings, countOf, summarySavings } from '../suggestionCopy'
export interface SavingsBannerMessageProps {
summary: SavingsSummary
}
export function SavingsBannerMessage({ summary }: SavingsBannerMessageProps): JSX.Element {
- const { time, bytes } = namedSaving(summary.saving, 1)
+ const savings = summarySavings(summary)
+ if (!savings) {
+ return <LegacySavingsMessage summary={summary} />
+ }
+ return (
+ <span className="flex flex-col gap-0.5">
+ {savings.waiting ? (
+ <span>
+ <span>People could wait </span>
+ <strong translate="no" className="text-success">
+ {savings.waiting}
+ </strong>
+ <span> less a month</span>
+ </span>
+ ) : (
+ <span translate="no">{savings.headline}</span>
+ )}
+ <span translate="no" className="text-xs text-secondary">
+ {savings.detail}
+ </span>
+ </span>
+ )
+}
+function LegacySavingsMessage({ summary }: SavingsBannerMessageProps): JSX.Element {
return (
<>
<strong translate="no">{countOf(summary.views, 'view', 'views')}</strong>
<span> could save </span>
- {time && (
- <>
+ {bannerSavings(summary).map(({ amount, label }, index) => (
+ <span key={label}>
+ {index > 0 && <span> and </span>}
<strong translate="no" className="text-success">
- {`~${time}`}
+ {amount}
</strong>
- <span> of query time</span>
- </>
- )}
- {time && bytes && <span> and </span>}
- {bytes && (
- <>
- <strong translate="no" className="text-success">
- {bytes}
- </strong>
- <span> of scanning</span>
- </>
- )}
+ <span> {label}</span>
+ </span>
+ ))}
<span> a month</span>
</>
)
diff --git a/products/warehouse_suggestions/frontend/components/SuggestedModelsSection.tsx b/products/warehouse_suggestions/frontend/components/SuggestedModelsSection.tsx
index 9c158f504b71..86c0ca88ec51 100644
--- a/products/warehouse_suggestions/frontend/components/SuggestedModelsSection.tsx
+++ b/products/warehouse_suggestions/frontend/components/SuggestedModelsSection.tsx
@@ -19,7 +19,7 @@ export function SuggestedModelsSection({ title, description, children }: Suggest
<span className="text-xs text-secondary">{description}</span>
</div>
<span className="hidden text-right text-xs text-secondary @min-[36rem]/suggestions:block">
- Saves a month
+ Main saving a month
</span>
</div>
<div className="divide-y divide-primary">{children}</div>
diff --git a/products/warehouse_suggestions/frontend/components/SuggestedModelsStrip.stories.tsx b/products/warehouse_suggestions/frontend/components/SuggestedModelsStrip.stories.tsx
index 7c3e81a1f30f..e3e6ecb55763 100644
--- a/products/warehouse_suggestions/frontend/components/SuggestedModelsStrip.stories.tsx
+++ b/products/warehouse_suggestions/frontend/components/SuggestedModelsStrip.stories.tsx
@@ -6,6 +6,7 @@ import { useStorybookMocks } from '~/mocks/browser'
import { DataModelingSyncInterval } from '~/types'
import type { WarehouseSuggestionApi, WarehouseSuggestionStatusApi } from '../generated/api.schemas'
+import type { MainSavingKind } from '../suggestionCopy'
import { warehouseSuggestionsLogic } from '../warehouseSuggestionsLogic'
import { SuggestedModelsStrip } from './SuggestedModelsStrip'
@@ -27,13 +28,25 @@ interface MaterializeOptions {
canAct?: boolean
unknownSources?: boolean
incremental?: boolean
+ backgroundSeconds?: number
+ readSeconds?: number
+ main?: MainSavingKind
+ savedBytes?: number
}
function materialize(
id: string,
name: string,
savedSeconds: number,
- { canAct = true, unknownSources = false, incremental = false }: MaterializeOptions = {}
+ {
+ canAct = true,
+ unknownSources = false,
+ incremental = false,
+ backgroundSeconds = 0,
+ readSeconds = 12,
+ main = 'waiting',
+ savedBytes = 3 * 10 ** 12,
+ }: MaterializeOptions = {}
): WarehouseSuggestionApi {
return {
id,
@@ -44,7 +57,11 @@ function materialize(
subject_name: name,
refresh_interval_seconds: 86400,
saves_seconds_per_month: savedSeconds,
- saves_bytes_per_month: 3 * 10 ** 12,
+ saves_bytes_per_month: savedBytes,
+ waiting_seconds_saved_per_month: savedSeconds - backgroundSeconds,
+ background_seconds_saved_per_month: backgroundSeconds,
+ main_saving: main,
+ read_seconds: readSeconds,
freshness_today_seconds: 3600,
freshness_after_seconds: 90000,
live_sources: { names: ['events'], hidden_count: 0 },
@@ -102,9 +119,13 @@ function unmaterialize(id: string, name: string): WarehouseSuggestionApi {
}
const suggestions = [
- materialize('1', 'customer_monthly_revenue', 1080),
- materialize('2', 'sessions_by_plan_daily', 2400),
- materialize('3', 'stripe_invoices_enriched', 0, { unknownSources: true }),
+ materialize('1', 'customer_monthly_revenue', 1080, { backgroundSeconds: -720, readSeconds: 45 }),
+ materialize('2', 'team_usage_rollup', 32400, { backgroundSeconds: 32400, main: 'background' }),
+ materialize('3', 'stripe_invoices_enriched', 600, {
+ unknownSources: true,
+ main: 'scanned',
+ savedBytes: 100 * 10 ** 12,
+ }),
unmaterialize('4', 'legacy_signup_funnel'),
]
diff --git a/products/warehouse_suggestions/frontend/components/SuggestedModelsStripContent.tsx b/products/warehouse_suggestions/frontend/components/SuggestedModelsStripContent.tsx
index d87a999a9056..c62656b11707 100644
--- a/products/warehouse_suggestions/frontend/components/SuggestedModelsStripContent.tsx
+++ b/products/warehouse_suggestions/frontend/components/SuggestedModelsStripContent.tsx
@@ -21,7 +21,7 @@ import { UnmaterializeSuggestionRow } from './UnmaterializeSuggestionRow'
const SECTION_COPY: Record<string, { title: string; description: string }> = {
materialize: {
title: 'Materialize',
- description: 'These views run their full query every time someone reads them.',
+ description: 'Store these views so reads and other refreshes can reuse their results.',
},
unmaterialize: {
title: 'Stop materializing',
diff --git a/products/warehouse_suggestions/frontend/components/SuggestionRow.tsx b/products/warehouse_suggestions/frontend/components/SuggestionRow.tsx
index de88e77fecac..2598c0ac9207 100644
--- a/products/warehouse_suggestions/frontend/components/SuggestionRow.tsx
+++ b/products/warehouse_suggestions/frontend/components/SuggestionRow.tsx
@@ -1,5 +1,7 @@
import { ReactNode } from 'react'
+import { Tooltip } from '@posthog/lemon-ui'
+
import { cn } from 'lib/utils/css-classes'
import type { WarehouseSuggestionApi } from '../generated/api.schemas'
@@ -56,9 +58,7 @@ export function SuggestionRow({
<div className="flex min-w-0">{title}</div>
{highlight && (
<div className="flex flex-wrap items-baseline gap-x-1.5 @min-[36rem]/suggestions:hidden">
- <span translate="no" className={AMOUNT_CLASS}>
- {highlight.amount}
- </span>
+ <HighlightAmount highlight={highlight} />
<span className="text-xs text-secondary">{highlight.narrowLabel}</span>
</div>
)}
@@ -84,9 +84,7 @@ export function SuggestionRow({
</div>
{highlight && (
<div className="hidden min-w-0 flex-col items-end @min-[36rem]/suggestions:flex">
- <span translate="no" className={AMOUNT_CLASS}>
- {highlight.amount}
- </span>
+ <HighlightAmount highlight={highlight} />
{highlight.detail && (
<span translate="no" className="text-xs tabular-nums text-secondary">
{highlight.detail}
@@ -105,3 +103,29 @@ export function SuggestionRow({
</div>
)
}
+
+function HighlightAmount({ highlight }: { highlight: SavingHighlight }): JSX.Element {
+ const amount = (
+ <span translate="no" className={AMOUNT_CLASS}>
+ {highlight.amount}
+ </span>
+ )
+ if (!highlight.tooltip?.length) {
+ return amount
+ }
+ return (
+ <Tooltip
+ title={
+ <div className="flex flex-col gap-0.5">
+ {highlight.tooltip.map((line) => (
+ <span key={line}>{line}</span>
+ ))}
+ </div>
+ }
+ >
+ <span tabIndex={0} className="cursor-help underline decoration-dotted underline-offset-2">
+ {amount}
+ </span>
+ </Tooltip>
+ )
+}
diff --git a/products/warehouse_suggestions/frontend/components/UnmaterializeSuggestionRow.tsx b/products/warehouse_suggestions/frontend/components/UnmaterializeSuggestionRow.tsx
index 3154e4a75c58..d8ebb4fdfad0 100644
--- a/products/warehouse_suggestions/frontend/components/UnmaterializeSuggestionRow.tsx
+++ b/products/warehouse_suggestions/frontend/components/UnmaterializeSuggestionRow.tsx
@@ -6,7 +6,7 @@ import { LemonButton } from '@posthog/lemon-ui'
import { iconForType } from '~/layout/panel-layout/ProjectTree/defaultTree'
import type { WarehouseSuggestionApi } from '../generated/api.schemas'
-import { isUnmaterializePayload, readSummary, savingHighlight, unmaterializeSaving } from '../suggestionCopy'
+import { isUnmaterializePayload, readSummary, unmaterializeHighlight } from '../suggestionCopy'
import { warehouseSuggestionsLogic } from '../warehouseSuggestionsLogic'
import { DismissSuggestionButtons } from './DismissSuggestionButtons'
import { SuggestionRow } from './SuggestionRow'
@@ -53,7 +53,7 @@ export function UnmaterializeSuggestionRow({
label={readSummary(suggestion.evidence)}
/>,
]}
- highlight={savingHighlight(unmaterializeSaving(payload))}
+ highlight={unmaterializeHighlight(payload)}
actions={
<>
<LemonButton
diff --git a/products/warehouse_suggestions/frontend/generated/api.schemas.ts b/products/warehouse_suggestions/frontend/generated/api.schemas.ts
index add9182a35f2..5b3d4ff8c873 100644
--- a/products/warehouse_suggestions/frontend/generated/api.schemas.ts
+++ b/products/warehouse_suggestions/frontend/generated/api.schemas.ts
@@ -60,6 +60,20 @@ export const WarehouseSuggestionRefreshIntervalBasisEnumApi = {
Savings: 'savings',
} as const
+/**
+ * * `waiting` - Waiting
+ * * `background` - Background
+ * * `scanned` - Scanned
+ */
+export type WarehouseSuggestionMainSavingEnumApi =
+ (typeof WarehouseSuggestionMainSavingEnumApi)[keyof typeof WarehouseSuggestionMainSavingEnumApi]
+
+export const WarehouseSuggestionMainSavingEnumApi = {
+ Waiting: 'waiting',
+ Background: 'background',
+ Scanned: 'scanned',
+} as const
+
export interface WarehouseSuggestionMaterializePayloadApi {
/** Sources that are always current, such as PostHog tables and direct connections. */
live_sources: WarehouseSuggestionVisibleSourcesApi
@@ -78,6 +92,12 @@ export interface WarehouseSuggestionMaterializePayloadApi {
* * `consumer_refresh` - Consumer refresh
* * `savings` - Savings */
refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasisEnumApi | null
+ /** The saving that earned the suggestion. waiting: people, agents and API callers wait less on reads. background: refreshes do less work. scanned: reads scan fewer bytes. Null on suggestions made before this was recorded.
+ *
+ * * `waiting` - Waiting
+ * * `background` - Background
+ * * `scanned` - Scanned */
+ main_saving: WarehouseSuggestionMainSavingEnumApi | null
/** Name of the view to materialize. */
subject_name: string
/** Proposed refresh interval, in seconds. */
@@ -104,6 +124,21 @@ export interface WarehouseSuggestionMaterializePayloadApi {
lookback_seconds?: number
/** How many materialized views compute this view again on every refresh, because it is not stored. 0 when the saving comes only from people's reads. */
recomputed_by_views?: number
+ /**
+ * Part of saves_seconds_per_month that people, agents and API callers spend waiting on reads of this view, in seconds. Null on suggestions made before this was recorded.
+ * @nullable
+ */
+ waiting_seconds_saved_per_month?: number | null
+ /**
+ * Part of saves_seconds_per_month that background refreshes spend, in seconds. Negative when refreshing the stored copy costs more background time than it saves. Null on suggestions made before this was recorded.
+ * @nullable
+ */
+ background_seconds_saved_per_month?: number | null
+ /**
+ * How long one read of the view takes today, in seconds. Null when no read was measured on its own.
+ * @nullable
+ */
+ read_seconds?: number | null
}
export interface WarehouseSuggestionUnmaterializePayloadApi {
diff --git a/products/warehouse_suggestions/frontend/suggestionCopy.test.ts b/products/warehouse_suggestions/frontend/suggestionCopy.test.ts
index af3e62ed684e..90977a44f9db 100644
--- a/products/warehouse_suggestions/frontend/suggestionCopy.test.ts
+++ b/products/warehouse_suggestions/frontend/suggestionCopy.test.ts
@@ -2,13 +2,19 @@ import type { WarehouseSuggestionMaterializePayloadApi } from './generated/api.s
import {
MonthlySaving,
SavingHighlight,
+ MainSavingKind,
+ SavingSplit,
+ SummarySavings,
UNKNOWN_SOURCES_CAVEAT,
dataAgeLabel,
deleteDisabledReason,
+ materializeHighlight,
notNowCondition,
savingHighlight,
savingSentence,
suggestedScheduleHint,
+ summarySavings,
+ unmaterializeHighlight,
todayAgeLabel,
} from './suggestionCopy'
@@ -32,6 +38,7 @@ function payload(
unknown_sources: unknownSources,
refresh_mode: 'full',
refresh_interval_basis: 'savings',
+ main_saving: null,
}
}
@@ -79,6 +86,118 @@ describe('suggestionCopy', () => {
expect(savingSentence(saving)).toEqual(sentence)
})
+ it.each<[string, MainSavingKind | null, number, number, SavingHighlight]>([
+ [
+ 'saves waiting but its refreshes cost background work',
+ 'waiting',
+ 2 * HOUR,
+ -HOUR,
+ {
+ amount: '~2 hours',
+ detail: 'less waiting',
+ narrowLabel: 'less waiting a month',
+ tooltip: ['~1 hour more background work a month', '~3.20 TB less scanned a month'],
+ },
+ ],
+ [
+ 'saves little waiting but a lot of scanning',
+ 'scanned',
+ 10 * 60,
+ 0,
+ {
+ amount: '~3.20 TB',
+ detail: 'less scanned',
+ narrowLabel: 'less scanned a month',
+ tooltip: ['~10 minutes less waiting a month'],
+ },
+ ],
+ [
+ 'saves only background work',
+ 'background',
+ 0,
+ 9 * HOUR,
+ {
+ amount: '~9 hours',
+ detail: 'less background work',
+ narrowLabel: 'less background work a month',
+ tooltip: ['~3.20 TB less scanned a month'],
+ },
+ ],
+ [
+ 'was made before the main saving was recorded',
+ null,
+ 0,
+ 0,
+ {
+ amount: '~20 minutes',
+ detail: '~3.20 TB less scanned',
+ narrowLabel: 'saved a month, ~3.20 TB less scanned',
+ },
+ ],
+ ])(
+ 'leads with the saving that earned a materialize suggestion that %s',
+ (_name, main, waiting, background, highlight) => {
+ const input = {
+ ...payload(null),
+ saves_bytes_per_month: 3.2 * TB,
+ waiting_seconds_saved_per_month: waiting,
+ background_seconds_saved_per_month: background,
+ main_saving: main,
+ }
+
+ expect(materializeHighlight(input)).toEqual(highlight)
+ }
+ )
+
+ it.each<[string, SavingSplit, number, SummarySavings]>([
+ [
+ 'both kinds of time are saved',
+ { waiting: 2 * HOUR, background: 9 * HOUR },
+ 3.2 * TB,
+ {
+ waiting: '~2 hours',
+ headline: null,
+ detail: 'Also ~9 hours less background work and ~3.20 TB less scanned, across 2 views',
+ },
+ ],
+ [
+ 'refreshes cost more background time than they save',
+ { waiting: 2 * HOUR, background: -3 * HOUR },
+ 0,
+ { waiting: '~2 hours', headline: null, detail: 'Also ~3 hours more background work, across 2 views' },
+ ],
+ [
+ 'stopping a materialization adds more waiting than the rest saves',
+ { waiting: -10 * 60, background: 2 * HOUR },
+ 0,
+ {
+ waiting: null,
+ headline: '~10 minutes more waiting and ~2 hours less background work a month',
+ detail: 'Across 2 views',
+ },
+ ],
+ ])('states each kind of time in the summary on its own when %s', (_name, split, bytes, expected) => {
+ expect(summarySavings({ views: 2, saving: { seconds: 0, bytes }, split })).toEqual(expected)
+ })
+
+ it('leads a stop-materializing row with the refresh time it saves and names the read time it adds', () => {
+ const stop = {
+ subject_name: 'orders',
+ refresh_seconds_per_month: 3 * HOUR,
+ refresh_bytes_per_month: 4 * TB,
+ live_seconds_per_month: 15 * 60,
+ live_bytes_per_month: TB,
+ read_seconds: 3.6,
+ }
+
+ expect(unmaterializeHighlight(stop)).toEqual({
+ amount: '~3 hours',
+ detail: 'less background work',
+ narrowLabel: 'less background work a month',
+ tooltip: ['~3.00 TB less scanned a month', 'Without the stored copy, reads would take ~15 minutes a month'],
+ })
+ })
+
it.each<[Parameters<typeof suggestedScheduleHint>[0], string]>([
['source_sync', "sources don't sync more often"],
['read_rate', "People don't read it more often"],
diff --git a/products/warehouse_suggestions/frontend/suggestionCopy.ts b/products/warehouse_suggestions/frontend/suggestionCopy.ts
index d7c170b9622a..85832603670e 100644
--- a/products/warehouse_suggestions/frontend/suggestionCopy.ts
+++ b/products/warehouse_suggestions/frontend/suggestionCopy.ts
@@ -1,3 +1,4 @@
+import { isNotNil } from 'lib/utils/guards'
import { humanizeBytes } from 'lib/utils/numbers'
import type {
@@ -30,12 +31,19 @@ const SURFACE_LABELS: Record<string, string> = {
app: 'Other pages',
unknown: 'Other',
}
+export const MAIN_SAVING_KINDS: MainSavingKind[] = ['waiting', 'background', 'scanned']
+const MAIN_SAVING_LABELS: Record<MainSavingKind, string> = {
+ waiting: 'less waiting',
+ background: 'less background work',
+ scanned: 'less scanned',
+}
export const UNKNOWN_SOURCES_CAVEAT = 'Some sources have no sync schedule, so their age is unknown.'
export interface SavingHighlight {
amount: string
detail: string | null
narrowLabel: string
+ tooltip?: string[]
}
export interface SurfaceSegment {
@@ -47,6 +55,30 @@ export interface SurfaceSegment {
export interface SavingsSummary {
views: number
saving: MonthlySaving
+ split: SavingSplit | null
+}
+
+export interface SavingSplit {
+ waiting: number
+ background: number
+}
+
+export interface BannerSaving {
+ amount: string
+ label: string
+}
+
+export interface SummarySavings {
+ waiting: string | null
+ headline: string | null
+ detail: string
+}
+
+export type MainSavingKind = NonNullable<WarehouseSuggestionMaterializePayloadApi['main_saving']>
+
+export interface MainSaving {
+ kind: MainSavingKind
+ amount: number
}
export interface MonthlySaving {
@@ -86,6 +118,18 @@ export function materializeSaving(payload: WarehouseSuggestionMaterializePayload
return { seconds: payload.saves_seconds_per_month, bytes: payload.saves_bytes_per_month }
}
+export function savingSplit(payload: WarehouseSuggestionApi['payload']): SavingSplit | null {
+ if (isUnmaterializePayload(payload)) {
+ return { waiting: -payload.live_seconds_per_month, background: payload.refresh_seconds_per_month }
+ }
+ if (!isMaterializePayload(payload)) {
+ return null
+ }
+ const waiting = payload.waiting_seconds_saved_per_month
+ const background = payload.background_seconds_saved_per_month
+ return waiting == null || background == null ? null : { waiting, background }
+}
+
export function unmaterializeSaving(payload: WarehouseSuggestionUnmaterializePayloadApi): MonthlySaving {
return {
seconds: Math.max(0, payload.refresh_seconds_per_month - payload.live_seconds_per_month),
@@ -166,6 +210,136 @@ export function savingHighlight(saving: MonthlySaving): SavingHighlight {
}
}
+export function mainSaving(payload: WarehouseSuggestionApi['payload']): MainSaving | null {
+ if (isMaterializePayload(payload)) {
+ return materializeMainSaving(payload)
+ }
+ return isUnmaterializePayload(payload) ? unmaterializeMainSaving(payload) : null
+}
+
+export function materializeHighlight(payload: WarehouseSuggestionMaterializePayloadApi): SavingHighlight {
+ if (!payload.main_saving) {
+ return savingHighlight(materializeSaving(payload))
+ }
+ const main = materializeMainSaving(payload)
+ const notes: Record<MainSavingKind, string | null> = {
+ waiting: waitingText(payload.waiting_seconds_saved_per_month ?? 0),
+ background: backgroundText(payload.background_seconds_saved_per_month ?? 0),
+ scanned: scannedText(payload.saves_bytes_per_month),
+ }
+ return headlineHighlight(
+ main,
+ MAIN_SAVING_KINDS.filter((kind) => kind !== main.kind).map((kind) => perMonth(notes[kind]))
+ )
+}
+
+export function unmaterializeHighlight(payload: WarehouseSuggestionUnmaterializePayloadApi): SavingHighlight {
+ const main = unmaterializeMainSaving(payload)
+ return headlineHighlight(main, [
+ perMonth(
+ main.kind === 'scanned'
+ ? backgroundText(payload.refresh_seconds_per_month)
+ : scannedText(unmaterializeSaving(payload).bytes)
+ ),
+ payload.live_seconds_per_month >= MIN_SECONDS_WORTH_NAMING
+ ? `Without the stored copy, reads would take ~${spokenDuration(payload.live_seconds_per_month, 1)} a month`
+ : null,
+ ])
+}
+
+export function materializeSavingLines(payload: WarehouseSuggestionMaterializePayloadApi): string[] {
+ const highlight = materializeHighlight(payload)
+ const readTime = payload.read_seconds != null ? `${readTimeLabel(payload.read_seconds)} today` : null
+ return [`${highlight.amount} ${highlight.narrowLabel}`, ...(highlight.tooltip ?? []), readTime].filter(isNotNil)
+}
+
+export function summarySavings(summary: SavingsSummary): SummarySavings | null {
+ if (!summary.split) {
+ return null
+ }
+ const { waiting, background } = summary.split
+ const views = countOf(summary.views, 'view', 'views')
+ const others = [backgroundText(background), scannedText(summary.saving.bytes)].filter(isNotNil)
+ if (waiting >= MIN_SECONDS_WORTH_NAMING) {
+ return {
+ waiting: `~${spokenDuration(waiting, 1)}`,
+ headline: null,
+ detail: others.length > 0 ? `Also ${others.join(' and ')}, across ${views}` : `Across ${views}`,
+ }
+ }
+ const parts = [waitingText(waiting), ...others].filter(isNotNil)
+ return parts.length > 0
+ ? { waiting: null, headline: `${parts.join(' and ')} a month`, detail: `Across ${views}` }
+ : null
+}
+
+export function bannerSavings(summary: SavingsSummary): BannerSaving[] {
+ const { time, bytes } = namedSaving(summary.saving, 1)
+ return [
+ time ? { amount: `~${time}`, label: 'of query time' } : null,
+ bytes ? { amount: bytes, label: 'of scanning' } : null,
+ ].filter(isNotNil)
+}
+
+function materializeMainSaving(payload: WarehouseSuggestionMaterializePayloadApi): MainSaving {
+ switch (payload.main_saving) {
+ case 'waiting':
+ return { kind: 'waiting', amount: payload.waiting_seconds_saved_per_month ?? 0 }
+ case 'background':
+ return { kind: 'background', amount: payload.background_seconds_saved_per_month ?? 0 }
+ case 'scanned':
+ return { kind: 'scanned', amount: payload.saves_bytes_per_month }
+ default:
+ return timeOrBytes(materializeSaving(payload), 'waiting')
+ }
+}
+
+function unmaterializeMainSaving(payload: WarehouseSuggestionUnmaterializePayloadApi): MainSaving {
+ const saving = unmaterializeSaving(payload)
+ if (!savesTime(saving)) {
+ return { kind: 'scanned', amount: saving.bytes }
+ }
+ return { kind: 'background', amount: payload.refresh_seconds_per_month }
+}
+
+function timeOrBytes(saving: MonthlySaving, timeKind: MainSavingKind): MainSaving {
+ return savesTime(saving) ? { kind: timeKind, amount: saving.seconds } : { kind: 'scanned', amount: saving.bytes }
+}
+
+function headlineHighlight(main: MainSaving, notes: (string | null)[]): SavingHighlight {
+ const amount = main.kind === 'scanned' ? humanizeBytes(main.amount) : spokenDuration(main.amount, 1)
+ const label = MAIN_SAVING_LABELS[main.kind]
+ return {
+ amount: `~${amount}`,
+ detail: label,
+ narrowLabel: `${label} a month`,
+ tooltip: notes.filter(isNotNil),
+ }
+}
+
+function perMonth(text: string | null): string | null {
+ return text && `${text} a month`
+}
+
+function waitingText(seconds: number): string | null {
+ return changeText(seconds, 'waiting')
+}
+
+function backgroundText(seconds: number): string | null {
+ return changeText(seconds, 'background work')
+}
+
+function changeText(seconds: number, noun: string): string | null {
+ if (Math.abs(seconds) < MIN_SECONDS_WORTH_NAMING) {
+ return null
+ }
+ return `~${spokenDuration(Math.abs(seconds), 1)} ${seconds > 0 ? 'less' : 'more'} ${noun}`
+}
+
+function scannedText(bytes: number): string | null {
+ return bytes >= MIN_BYTES_WORTH_NAMING ? `~${humanizeBytes(bytes)} less scanned` : null
+}
+
export function savingSentence(saving: MonthlySaving): string {
const { time, bytes } = namedSaving(saving, 1)
const parts = [time && `${time} of query time`, bytes && `${bytes} of scanning`].filter(Boolean)
diff --git a/products/warehouse_suggestions/frontend/warehouseSuggestionsLogic.test.ts b/products/warehouse_suggestions/frontend/warehouseSuggestionsLogic.test.ts
index 15940bba7b3a..4db00d56bb83 100644
--- a/products/warehouse_suggestions/frontend/warehouseSuggestionsLogic.test.ts
+++ b/products/warehouse_suggestions/frontend/warehouseSuggestionsLogic.test.ts
@@ -73,6 +73,7 @@ const MATERIALIZE: WarehouseSuggestionApi = {
unknown_sources: { names: [], hidden_count: 0 },
refresh_mode: 'full',
refresh_interval_basis: 'savings',
+ main_saving: null,
},
payload_version: 1,
evidence: { human_requests: 200, human_users: 6, human_days: 25 },
@@ -102,6 +103,19 @@ function materialize(id: string, savesSeconds: number, savesBytes = 0): Warehous
}
}
+function ledBy(id: string, main: 'waiting' | 'scanned', amount: number): WarehouseSuggestionApi {
+ const suggestion = materialize(id, main === 'waiting' ? amount : 0, main === 'scanned' ? amount : 0)
+ return {
+ ...suggestion,
+ payload: {
+ ...suggestion.payload,
+ main_saving: main,
+ waiting_seconds_saved_per_month: main === 'waiting' ? amount : 0,
+ background_seconds_saved_per_month: 0,
+ },
+ }
+}
+
const INCREMENTAL: WarehouseSuggestionApi = {
...MATERIALIZE,
id: 'suggestion-incremental',
@@ -526,7 +540,29 @@ describe('warehouseSuggestionsLogic', () => {
await mountLogic()
- expect(logic.values.savingsSummary).toEqual({ views: 3, saving: { seconds: 3000, bytes: 5e12 } })
+ expect(logic.values.savingsSummary).toEqual({
+ views: 3,
+ saving: { seconds: 3000, bytes: 5e12 },
+ split: null,
+ })
+ })
+
+ it('lists rows by the saving they lead with, so equal units sit together and larger comes first', async () => {
+ mockList.mockResolvedValue(
+ page([
+ ledBy('scanned', 'scanned', 100e12),
+ ledBy('ten-minutes', 'waiting', 600),
+ ledBy('half-hour', 'waiting', 1800),
+ ])
+ )
+
+ await mountLogic()
+
+ expect(logic.values.listedSuggestions.map((suggestion) => suggestion.id)).toEqual([
+ 'half-hour',
+ 'ten-minutes',
+ 'scanned',
+ ])
})
it('drops an accepted row from the counts at once and from the list after it leaves', async () => {
diff --git a/products/warehouse_suggestions/frontend/warehouseSuggestionsLogic.ts b/products/warehouse_suggestions/frontend/warehouseSuggestionsLogic.ts
index ca16cb431189..4f66207957cd 100644
--- a/products/warehouse_suggestions/frontend/warehouseSuggestionsLogic.ts
+++ b/products/warehouse_suggestions/frontend/warehouseSuggestionsLogic.ts
@@ -18,6 +18,7 @@ import posthog from 'posthog-js'
import { ApiConfig } from 'lib/api'
import { ApiError } from 'lib/api-error'
import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
+import { isNotNil } from 'lib/utils/guards'
import { EMPTY_INCREMENTAL_DRAFT, IncrementalConfigDraft } from 'scenes/data-warehouse/editor/IncrementalConfigFields'
import { DataModelingSyncInterval } from '~/types'
@@ -47,12 +48,13 @@ import type { WarehouseSuggestionDismissalReasonEnumApi } from './generated/api.
import { INTERVAL_SECONDS, intervalForSeconds } from './refreshIntervals'
import { subjectUrl } from './subjectUrl'
import {
- MonthlySaving,
+ MAIN_SAVING_KINDS,
SavingsSummary,
isMaterializePayload,
+ mainSaving,
monthlySaving,
notNowCondition,
- savesTime,
+ savingSplit,
} from './suggestionCopy'
export type StripState = 'loading' | 'error' | 'hidden' | 'warming_up' | 'not_eligible' | 'active' | 'all_done'
@@ -114,11 +116,11 @@ function assignedMessage({ assignee }: WarehouseSuggestionApi): string {
}
function suggestionWorth(suggestion: WarehouseSuggestionApi): [number, number] {
- const saving = monthlySaving(suggestion.payload)
- if (!saving) {
+ const main = mainSaving(suggestion.payload)
+ if (!main) {
return [KIND_ORDER[suggestion.kind] ?? 0, 0]
}
- return savesTime(saving) ? [0, -saving.seconds] : [1, -saving.bytes]
+ return [MAIN_SAVING_KINDS.indexOf(main.kind), -main.amount]
}
function compareSuggestions(a: WarehouseSuggestionApi, b: WarehouseSuggestionApi): number {
@@ -657,15 +659,24 @@ export const warehouseSuggestionsLogic: LogicWrapper<warehouseSuggestionsLogicTy
savingsSummary: [
(s) => [s.surfaceSuggestions],
(open: WarehouseSuggestionApi[]): SavingsSummary => {
- const savings = open
- .map(({ payload }) => monthlySaving(payload))
- .filter((saving): saving is MonthlySaving => saving !== null)
+ const savings = open.flatMap(({ payload }) => {
+ const saving = monthlySaving(payload)
+ return saving ? [{ saving, split: savingSplit(payload) }] : []
+ })
+ const splits = savings.map((entry) => entry.split).filter(isNotNil)
return {
views: savings.length,
saving: {
- seconds: sum(savings.map((saving) => saving.seconds)),
- bytes: sum(savings.map((saving) => saving.bytes)),
+ seconds: sum(savings.map((entry) => entry.saving.seconds)),
+ bytes: sum(savings.map((entry) => entry.saving.bytes)),
},
+ split:
+ splits.length === savings.length
+ ? {
+ waiting: sum(splits.map((split) => split.waiting)),
+ background: sum(splits.map((split) => split.background)),
+ }
+ : null,
}
},
],
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 8b0ca8a387ee..874949775bf2 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -85170,6 +85170,20 @@ export namespace Schemas {
Savings: 'savings',
} as const;
+ /**
+ * * `waiting` - Waiting
+ * * `background` - Background
+ * * `scanned` - Scanned
+ */
+ export type WarehouseSuggestionMainSavingEnum = typeof WarehouseSuggestionMainSavingEnum[keyof typeof WarehouseSuggestionMainSavingEnum];
+
+
+ export const WarehouseSuggestionMainSavingEnum = {
+ Waiting: 'waiting',
+ Background: 'background',
+ Scanned: 'scanned',
+ } as const;
+
export interface WarehouseSuggestionMaterializePayload {
/** Sources that are always current, such as PostHog tables and direct connections. */
live_sources: WarehouseSuggestionVisibleSources;
@@ -85188,6 +85202,12 @@ export namespace Schemas {
* * `consumer_refresh` - Consumer refresh
* * `savings` - Savings */
refresh_interval_basis: WarehouseSuggestionRefreshIntervalBasisEnum | null;
+ /** The saving that earned the suggestion. waiting: people, agents and API callers wait less on reads. background: refreshes do less work. scanned: reads scan fewer bytes. Null on suggestions made before this was recorded.
+ *
+ * * `waiting` - Waiting
+ * * `background` - Background
+ * * `scanned` - Scanned */
+ main_saving: WarehouseSuggestionMainSavingEnum | null;
/** Name of the view to materialize. */
subject_name: string;
/** Proposed refresh interval, in seconds. */
@@ -85214,6 +85234,21 @@ export namespace Schemas {
lookback_seconds?: number;
/** How many materialized views compute this view again on every refresh, because it is not stored. 0 when the saving comes only from people's reads. */
recomputed_by_views?: number;
+ /**
+ * Part of saves_seconds_per_month that people, agents and API callers spend waiting on reads of this view, in seconds. Null on suggestions made before this was recorded.
+ * @nullable
+ */
+ waiting_seconds_saved_per_month?: number | null;
+ /**
+ * Part of saves_seconds_per_month that background refreshes spend, in seconds. Negative when refreshing the stored copy costs more background time than it saves. Null on suggestions made before this was recorded.
+ * @nullable
+ */
+ background_seconds_saved_per_month?: number | null;
+ /**
+ * How long one read of the view takes today, in seconds. Null when no read was measured on its own.
+ * @nullable
+ */
+ read_seconds?: number | null;
}
export interface WarehouseSuggestionUnmaterializePayload {