:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(today): publish ready numbers before all sources finish (#127509)
Shy Alter100e7516aa00
diff --git a/products/customer_analytics/backend/facade/api.py b/products/customer_analytics/backend/facade/api.pyindex 5706ad38a45e..bf9e3b57ec94 100644--- a/products/customer_analytics/backend/facade/api.py+++ b/products/customer_analytics/backend/facade/api.py@@ -5952,3 +5952,7 @@ def list_announcement_channels(team_id: int) -> list[contracts.AnnouncementChanndef get_key_metrics(*, team: Team, user: "User") -> list[contracts.KeyMetric] | None:"""The key metrics, or None while a background calculation is still running."""return _key_metrics_logic.key_metrics(team, user)+++def get_key_metric_names(*, team: Team) -> dict[str, str]:+ return _key_metrics_logic.key_metric_names(team)diff --git a/products/customer_analytics/backend/logic/key_metrics.py b/products/customer_analytics/backend/logic/key_metrics.pyindex 8325098d37a0..a8f3bc0d1a4e 100644--- a/products/customer_analytics/backend/logic/key_metrics.py+++ b/products/customer_analytics/backend/logic/key_metrics.py@@ -56,6 +56,11 @@ def _action_ids(nodes: dict[str, dict[str, Any]]) -> list[int]:return [int(node["id"]) for node in nodes.values() if node["kind"] == "ActionsNode"]+def key_metric_names(team: Team) -> dict[str, str]:+ nodes = _configured_nodes(team)+ return {key: node.get("custom_name") or name for key, name in KEY_METRICS.items() if (node := nodes.get(key))}++def _anchored(filters: Any, team: Team, anchor: datetime) -> Any:if isinstance(filters, list):return [_anchored(item, team, anchor) for item in filters]diff --git a/products/product_analytics/backend/facade/api.py b/products/product_analytics/backend/facade/api.pyindex b7d3493328fd..bc104f537183 100644--- a/products/product_analytics/backend/facade/api.py+++ b/products/product_analytics/backend/facade/api.py@@ -291,6 +291,40 @@ def save_saved_insight_query(return None+def viewable_insight_definitions(+ *, team: Team, user: User, short_ids: Collection[str]+) -> dict[str, SavedInsightDefinition]:+ access_control = UserAccessControl(user=user, team=team, organization_id=str(team.organization_id))+ insights = list(Insight.objects.filter(team__project_id=team.project_id, short_id__in=short_ids, deleted=False))+ denied = set(+ access_control.denied_object_ids(+ "insight", ((str(insight.pk), insight.team_id, insight.created_by_id) for insight in insights), "viewer"+ )+ )+ return {+ insight.short_id: SavedInsightDefinition(+ id=insight.pk,+ short_id=insight.short_id,+ name=insight.name or insight.derived_name,+ query=insight.query or {},+ )+ for insight in insights+ if str(insight.pk) not in denied+ }+++def insight_query_complete(*, team: Team, query_id: str) -> bool:+ from products.product_analytics.backend import insight_series++ return insight_series.query_complete(team=team, query_id=query_id)+++def insight_series_supported(*, query: dict[str, Any], dashboard: InsightDashboardContext | None = None) -> bool:+ from products.product_analytics.backend import insight_series++ return insight_series.supports_series(query=query, dashboard=dashboard)++def get_insight_series(*,team: Team,@@ -299,12 +333,15 @@ def get_insight_series(dashboard: InsightDashboardContext | None = None,blocking: bool = False,wait_until: float | None = None,+ raise_on_pending: bool = False,) -> InsightSeries | None:insight = Insight.objects.filter(team__project_id=team.project_id, short_id=short_id, deleted=False).first()if insight is None:return Noneaccess_control = UserAccessControl(user=user, team=team, organization_id=str(team.organization_id))- if not access_control.check_access_level_for_object(insight, "viewer"):+ if access_control.denied_object_ids(+ "insight", [(str(insight.pk), insight.team_id, insight.created_by_id)], "viewer"+ ):return Nonefrom products.product_analytics.backend import ( # noqa: PLC0415 -- insight_series imports the query runner, which imports this facadeinsight_series,@@ -319,4 +356,5 @@ def get_insight_series(dashboard=dashboard,blocking=blocking,wait_until=wait_until,+ raise_on_pending=raise_on_pending,)diff --git a/products/product_analytics/backend/facade/contracts.py b/products/product_analytics/backend/facade/contracts.pyindex 82977a4c7e9a..4a4d925d4592 100644--- a/products/product_analytics/backend/facade/contracts.py+++ b/products/product_analytics/backend/facade/contracts.py@@ -73,6 +73,12 @@ class SavedInsightDefinition:query: dict[str, Any]+class InsightSeriesNotReady(Exception):+ def __init__(self, query_id: str | None = None) -> None:+ super().__init__(query_id)+ self.query_id = query_id++@dataclass(frozen=True)class InsightDataCheckFilter:"""A rule in the shape the project's internal and test users filter stores it."""diff --git a/products/product_analytics/backend/insight_series.py b/products/product_analytics/backend/insight_series.pyindex 1a1ebe4afb8f..99b19cab466c 100644--- a/products/product_analytics/backend/insight_series.py+++ b/products/product_analytics/backend/insight_series.py@@ -20,7 +20,11 @@ from posthog.models import Team, Userfrom posthog.schema_migrations.upgrade import upgradefrom posthog.week_start_day import WeekStartDay-from products.product_analytics.backend.facade.contracts import InsightDashboardContext, InsightSeries+from products.product_analytics.backend.facade.contracts import (+ InsightDashboardContext,+ InsightSeries,+ InsightSeriesNotReady,+)ADDITIVE_MATH = {"total",@@ -47,6 +51,37 @@ EXECUTION_MODE = ExecutionMode.RECENT_CACHE_CALCULATE_ASYNC_IF_STALEBLOCKING_EXECUTION_MODE = ExecutionMode.RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE+def query_complete(*, team: Team, query_id: str) -> bool:+ status = get_query_status(team_id=team.pk, query_id=query_id)+ if status.error:+ raise ValueError("Insight query failed")+ return bool(status.complete)+++def _supports_series(query: dict[str, Any], filters: dict[str, Any]) -> bool:+ source = query.get("source", query)+ if query.get("kind") == "DataVisualizationNode" and source.get("kind") == "HogQLQuery":+ settings = query.get("chartSettings") or {}+ return bool(+ query.get("display") in CHARTED_SQL_DISPLAYS+ and (settings.get("xAxis") or {}).get("column")+ and any(axis.get("column") for axis in settings.get("yAxis") or [])+ )+ if source.get("kind") != "TrendsQuery" or (source.get("trendsFilter") or {}).get("display") in UNCHARTED_DISPLAYS:+ return False+ interval = filters.get("interval") or source.get("interval") or "day"+ if interval not in LOOKBACK:+ interval = "day"+ return interval != "day" or not (source.get("dateRange") or {}).get("daysOfWeek")+++def supports_series(*, query: dict[str, Any], dashboard: InsightDashboardContext | None = None) -> bool:+ if dashboard is None:+ return _supports_series(query, {})+ effective = resolve_effective_dashboard_filters(query, dashboard.filters, dashboard.tile_filters)+ return _supports_series(effective.query, effective.filters or {})++def _run(team: Team,user: User,@@ -55,6 +90,7 @@ def _run(variables: dict[str, Any] | None,blocking: bool = False,wait_until: float | None = None,+ raise_on_pending: bool = False,) -> dict[str, Any]:from posthog.api.services.query import process_query_dict # noqa: PLC0415@@ -67,6 +103,13 @@ def _run(user=user,)result = response.model_dump() if isinstance(response, BaseModel) else response+ status = result.get("query_status") or {}+ if raise_on_pending and (result.get("error") or status.get("error")):+ raise ValueError("Insight query failed")+ if raise_on_pending and result.get("results") is None:+ if not status.get("id"):+ raise ValueError("Insight query returned no results or query status")+ raise InsightSeriesNotReady(status["id"])if blocking and wait_until is not None and result.get("results") is None:status = result.get("query_status") or {}while status and not status.get("complete") and not status.get("error"):@@ -143,13 +186,12 @@ def _sql_series(variables: dict[str, Any] | None,blocking: bool = False,wait_until: float | None = None,+ raise_on_pending: bool = False,) -> InsightSeries | None:settings = query.get("chartSettings") or {}x_column = (settings.get("xAxis") or {}).get("column")y_axes = [axis for axis in settings.get("yAxis") or [] if axis.get("column")]- if query.get("display") not in CHARTED_SQL_DISPLAYS or not x_column or not y_axes:- return None- response = _run(team, user, query["source"], filters, variables, blocking, wait_until)+ response = _run(team, user, query["source"], filters, variables, blocking, wait_until, raise_on_pending)if response.get("hasMore"):return Noneposition = {column: index for index, column in enumerate(response.get("columns") or [])}@@ -219,6 +261,7 @@ def insight_series(dashboard: InsightDashboardContext | None,blocking: bool = False,wait_until: float | None = None,+ raise_on_pending: bool = False,) -> InsightSeries | None:dashboard_filters: dict[str, Any] = {}variables: dict[str, Any] | None = None@@ -228,6 +271,8 @@ def insight_series(dashboard_filters = effective.filters or {}variables = dashboard.variablesfilters = {key: value for key, value in dashboard_filters.items() if key not in DATE_FILTERS}+ if not _supports_series(query, dashboard_filters):+ return Nonesource = query.get("source", query)if query.get("kind") == "DataVisualizationNode" and source.get("kind") == "HogQLQuery":return _sql_series(@@ -240,16 +285,13 @@ def insight_series(variables=variables,blocking=blocking,wait_until=wait_until,+ raise_on_pending=raise_on_pending,)trends = source.get("trendsFilter") or {}- if source.get("kind") != "TrendsQuery" or trends.get("display") in UNCHARTED_DISPLAYS:- return Noneinterval = dashboard_filters.get("interval") or source.get("interval") or "day"if interval not in LOOKBACK:interval = "day"date_range = source.get("dateRange") or {}- if interval == "day" and date_range.get("daysOfWeek"):- return Nonekept = {key: value for key, value in date_range.items() if key not in DATE_FILTERS}bounded = {key: value for key, value in source.items() if key != "compareFilter"}bounded.update(@@ -257,7 +299,7 @@ def insight_series(dateRange={**kept, "date_from": LOOKBACK[interval], "excludeIncompletePeriods": True},trendsFilter={**{k: v for k, v in trends.items() if k != "smoothingIntervals"}, "metricShowChange": False},)- response = _run(team, user, bounded, filters, variables, blocking, wait_until)+ response = _run(team, user, bounded, filters, variables, blocking, wait_until, raise_on_pending)formula = bool(trends.get("formula") or trends.get("formulas") or trends.get("formulaNodes"))several = len(source.get("series") or []) > 1# `_run` executes the upgraded query, which stamps `kind` on series saved without one.diff --git a/products/product_analytics/backend/tests/test_insight_series.py b/products/product_analytics/backend/tests/test_insight_series.pyindex 552515131e61..2e36caaa0158 100644--- a/products/product_analytics/backend/tests/test_insight_series.py+++ b/products/product_analytics/backend/tests/test_insight_series.py@@ -20,13 +20,55 @@ from posthog.week_start_day import WeekStartDayfrom products.access_control.backend.facade.testing import create_access_controlfrom products.actions.backend.models.action import Actionfrom products.product_analytics.backend import insight_series-from products.product_analytics.backend.facade.api import get_insight_series-from products.product_analytics.backend.facade.contracts import InsightDashboardContext+from products.product_analytics.backend.facade.api import get_insight_series, viewable_insight_definitions+from products.product_analytics.backend.facade.contracts import InsightDashboardContext, InsightSeriesNotReadyfrom products.product_analytics.backend.insight_series import _visiblefrom products.product_analytics.backend.models.insight import Insightclass TestBoundedInsightQuery(SimpleTestCase):+ @parameterized.expand([(False, False), (True, False), (True, True)])+ def test_query_status_is_scoped_and_does_not_prepare_the_query(self, complete: bool, error: bool) -> None:+ with (+ patch.object(+ insight_series,+ "get_query_status",+ return_value=QueryStatus(id="q", team_id=7, complete=complete, error=error),+ ) as status,+ patch("posthog.api.services.query.process_query_dict") as prepare,+ ):+ if error:+ with self.assertRaises(ValueError):+ insight_series.query_complete(team=cast(Team, SimpleNamespace(pk=7)), query_id="q")+ else:+ self.assertEqual(+ insight_series.query_complete(team=cast(Team, SimpleNamespace(pk=7)), query_id="q"), complete+ )+ status.assert_called_once_with(team_id=7, query_id="q")+ prepare.assert_not_called()++ @parameterized.expand(+ [+ ("pending", {"query_status": {"id": "q", "complete": False}}, InsightSeriesNotReady),+ ("error", {"query_status": {"id": "q", "error": True}}, ValueError),+ ("empty", {"results": []}, None),+ ("top_level_error", {"results": None, "error": "Invalid query"}, ValueError),+ ("missing_status", {"results": None}, ValueError),+ ]+ )+ def test_pending_is_distinct_from_empty_and_error(+ self, _name: str, response: dict[str, Any], error: type[Exception] | None+ ) -> None:+ with patch("posthog.api.services.query.process_query_dict", return_value=response):+ if error:+ with self.assertRaises(error):+ insight_series._run(cast(Team, None), cast(User, None), {}, None, None, raise_on_pending=True)+ else:+ self.assertEqual(+ insight_series._run(cast(Team, None), cast(User, None), {}, None, None, raise_on_pending=True),+ response,+ )+@parameterized.expand([("cached", True, False, False, 0.0, 0),@@ -555,7 +597,8 @@ class TestGetInsightSeries(ClickhouseTestMixin, BaseTest):self.assertEqual((found, enqueue.call_count), (None, 1))- def test_an_insight_the_person_cannot_view_returns_none(self) -> None:+ @parameterized.expand([False, True])+ def test_an_insight_the_person_cannot_view_returns_none(self, from_child_environment: bool) -> None:self.organization.available_product_features = [{"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}]@@ -565,5 +608,24 @@ class TestGetInsightSeries(ClickhouseTestMixin, BaseTest):team_id=self.team.id, resource="insight", resource_id=str(insight.id), access_level="none")reader = self._create_user("[email protected]")+ requesting_team = (+ Team.objects.create(+ organization=self.organization,+ project_id=self.team.project_id,+ parent_team=self.team,+ name="Other environment",+ )+ if from_child_environment+ else self.team+ )- self.assertIsNone(get_insight_series(team=self.team, user=reader, short_id=insight.short_id))+ self.assertIsNone(get_insight_series(team=requesting_team, user=reader, short_id=insight.short_id))+ self.assertEqual(+ viewable_insight_definitions(team=requesting_team, user=reader, short_ids=[insight.short_id]), {}+ )+ self.assertEqual(+ viewable_insight_definitions(team=requesting_team, user=self.user, short_ids=[insight.short_id])[+ insight.short_id+ ].name,+ insight.name or insight.derived_name,+ )diff --git a/products/today/backend/facade/api.py b/products/today/backend/facade/api.pyindex 46fe985f956b..1600ab931910 100644--- a/products/today/backend/facade/api.py+++ b/products/today/backend/facade/api.py@@ -52,8 +52,10 @@ def report_figure_marks(*, team: Team, user: User, report_id: str) -> list[contrreturn report_pages.figure_marks(page, artefacts, _jev(team, user, figure_sources.FIGURE_MODEL))-def get_highlights(*, team: Team, user: User, scopes: frozenset[str] | None) -> contracts.Highlights:- return highlights.get_highlights(team=team, user=user, scopes=scopes)+def get_highlights(+ *, team: Team, user: User, scopes: frozenset[str] | None, refresh: bool = False+) -> contracts.Highlights:+ return highlights.get_highlights(team=team, user=user, scopes=scopes, refresh=refresh)def get_pulse_pins(*, team: Team, user: User, scopes: frozenset[str] | None) -> contracts.PulsePins:diff --git a/products/today/backend/facade/contracts.py b/products/today/backend/facade/contracts.pyindex 46cac6e6a5c0..6f001fc02219 100644--- a/products/today/backend/facade/contracts.py+++ b/products/today/backend/facade/contracts.py@@ -22,6 +22,7 @@ from .enums import (HighlightUnit,ImpactNumberKey,KeyClauseRole,+ PulseSlotStatus,)if TYPE_CHECKING:@@ -227,6 +228,16 @@ class HighlightPin:status: HighlightPinStatus+@frozen+class PulseSlot:+ key: str+ name: str+ reason: HighlightMetricReason | None = None+ status: PulseSlotStatus = PulseSlotStatus.PENDING+ url: str | None = None+ interval: HighlightInterval | None = None++@dataclass(frozen=True)class Highlights:status: HighlightsStatus@@ -236,6 +247,7 @@ class Highlights:pulse: list[HighlightMetric]items: list[HighlightItem]pins: list[HighlightPin] = field(default_factory=list)+ pulse_slots: list[PulseSlot] | None = Noneclass DailyValueNotReady(Exception):diff --git a/products/today/backend/facade/enums.py b/products/today/backend/facade/enums.pyindex 98b970a16dba..038f0359c10b 100644--- a/products/today/backend/facade/enums.py+++ b/products/today/backend/facade/enums.py@@ -65,6 +65,13 @@ class HighlightPinStatus(LabeledStrEnum):UNAVAILABLE = "unavailable", "Unavailable"+class PulseSlotStatus(LabeledStrEnum):+ PENDING = "pending", "Loading"+ READY = "ready", "Ready"+ EMPTY = "empty", "No data"+ ERROR = "error", "Could not load"++class HighlightInterval(LabeledStrEnum):DAY = "day", "Day"WEEK = "week", "Week"diff --git a/products/today/backend/feature_flags.py b/products/today/backend/feature_flags.pyindex 8c42891718ce..5c02ba01e0d7 100644--- a/products/today/backend/feature_flags.py+++ b/products/today/backend/feature_flags.py@@ -9,6 +9,11 @@ logger = structlog.get_logger(__name__)TODAY_RAIL_NAV_FLAG = "today-rail-nav"TODAY_REPORT_JEV_FLAG = "today-report-jev"+TODAY_PROGRESSIVE_PULSE_FLAG = "today-progressive-pulse"+++def progressive_pulse_enabled(user: User, team: Team) -> bool:+ return _flag_on(TODAY_PROGRESSIVE_PULSE_FLAG, user, team)def is_enabled_for(user: User, team: Team) -> bool:diff --git a/products/today/backend/logic/highlight_types.py b/products/today/backend/logic/highlight_types.pyindex 06acfd95c1a9..6daa57233251 100644--- a/products/today/backend/logic/highlight_types.py+++ b/products/today/backend/logic/highlight_types.py@@ -27,6 +27,7 @@ from ..facade.contracts import (HighlightContext,HighlightFindings,HighlightSource,+ PulseSlot,)# Short names for Today's own sources in today_highlights/. Other products use the facade contract.@@ -57,12 +58,14 @@ class Context:today: date,scopes: frozenset[str] | None = None,pinned: frozenset[str] = frozenset(),+ publish_pulse: Callable[[HighlightFindings, list[PulseSlot]], None] | None = None,) -> None:self.team = teamself.user = userself.today = todayself.scopes = scopesself.pinned = pinned+ self.publish_pulse = publish_pulseself.failures: list[str] = []self.access = UserAccessControl(user=user, team=team)diff --git a/products/today/backend/logic/highlights.py b/products/today/backend/logic/highlights.pyindex 7bfb8734a2d3..321ec97d2bfe 100644--- a/products/today/backend/logic/highlights.py+++ b/products/today/backend/logic/highlights.py@@ -9,6 +9,7 @@ from uuid import UUIDfrom django.conf import settingsfrom django.core.cache import cache+from django.db import transactionfrom django.utils import timezoneimport structlog@@ -17,6 +18,7 @@ from pydantic import TypeAdapter, ValidationErrorfrom temporalio.common import WorkflowIDReusePolicyfrom posthog.clickhouse.query_tagging import Feature, Product, tags_context+from posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exceptionfrom posthog.models import Team, Userfrom posthog.temporal.common.client import sync_connect@@ -31,7 +33,9 @@ from ..facade.enums import (HighlightPinStatus,HighlightPriority,HighlightsStatus,+ PulseSlotStatus,)+from ..feature_flags import progressive_pulse_enabledfrom ..models import TodayHighlightsfrom ..temporal.inputs import (HIGHLIGHTS_WORKFLOW_NAME,@@ -62,10 +66,24 @@ _FINDINGS = TypeAdapter(HighlightFindings)_HIGHLIGHTS = TypeAdapter(contracts.Highlights)-def get_highlights(*, team: Team, user: User, scopes: frozenset[str] | None) -> contracts.Highlights:+@frozen+class PulseSourceProgress:+ found: HighlightFindings+ slots: list[contracts.PulseSlot]+++_PROGRESS = TypeAdapter(dict[str, PulseSourceProgress])+++def get_highlights(+ *, team: Team, user: User, scopes: frozenset[str] | None, refresh: bool = False+) -> contracts.Highlights:now = timezone.now()day = now.astimezone(team.timezone_info).date()key = _key(team, user, scopes)+ progressive = progressive_pulse_enabled(user, team)+ if progressive:+ key = hashlib.sha256(f"{key}:progressive".encode()).hexdigest()[:32]rows = (TodayHighlights.objects.for_team(team.id).filter(user_id=user.id, key=key, local_day=day)@@ -73,8 +91,16 @@ def get_highlights(*, team: Team, user: User, scopes: frozenset[str] | None) ->)latest = rows.first()# A row with a pins key puts pinned tiles first, so compute it again with suggestions only.- if latest is None or latest.pins_key or _is_due(latest, now):- _start(team, user, scopes, day, key, after=latest.id if latest else None)+ retry_failed = (+ progressive+ and refresh+ and latest is not None+ and latest.failed_sources+ and latest.status != HighlightsStatus.COMPUTING+ and cache.add(f"today_highlights:retry:{team.id}:{user.id}", True, FAILED_CACHE_SECONDS)+ )+ if latest is None or latest.pins_key or _is_due(latest, now) or retry_failed:+ _start(team, user, scopes, day, key, after=latest.id if latest else None, progressive=progressive)status = HighlightsStatus.COMPUTINGelse:status = HighlightsStatus(latest.status)@@ -83,11 +109,34 @@ def get_highlights(*, team: Team, user: User, scopes: frozenset[str] | None) ->if latest is not None and latest.status == HighlightsStatus.READYelse rows.filter(status=HighlightsStatus.READY).first())+ if progressive and latest is not None and latest.content.get("pulse_slots"):+ with suppress(ValidationError):+ content = _HIGHLIGHTS.validate_python(latest.content)+ if status == HighlightsStatus.COMPUTING and ready is not None and ready.id != latest.id:+ old = _HIGHLIGHTS.validate_python(ready.content)+ pending = {slot.key for slot in content.pulse_slots or [] if slot.status == PulseSlotStatus.PENDING}+ retained = [metric for metric in old.pulse if metric.key in pending]+ retained_keys = {metric.key for metric in retained}+ content = replace(+ content,+ pulse=[*content.pulse, *retained],+ pulse_slots=[+ replace(slot, status=PulseSlotStatus.READY) if slot.key in retained_keys else slot+ for slot in content.pulse_slots or []+ ],+ )+ return replace(content, status=status)if ready is not None:with suppress(ValidationError):return replace(_HIGHLIGHTS.validate_python(ready.content), status=status)return contracts.Highlights(- status=status, generated_at=None, day=day, refresh_at=_refresh_at(team, day), pulse=[], items=[]+ status=status,+ generated_at=None,+ day=day,+ refresh_at=_refresh_at(team, day),+ pulse=[],+ items=[],+ pulse_slots=[] if progressive else None,)@@ -111,6 +160,7 @@ def _start(day: date,key: str,after: UUID | None,+ progressive: bool = False,) -> None:lock = f"today_highlights:start:{team.id}:{user.id}:{key}:{day.isoformat()}:{after}"if not cache.add(lock, True, START_LOCK_SECONDS):@@ -123,6 +173,7 @@ def _start(local_day=day.isoformat(),scopes=None if scopes is None else sorted(scopes),source_names=[source.name for source in load_sources()],+ progressive=progressive,)try:asyncio.run(@@ -139,6 +190,84 @@ def _start(capture_exception(error, {"highlights_id": str(row.id), "product": "today"})+def _pulse_content(+ content: contracts.Highlights,+ progress: dict[str, PulseSourceProgress],+ previous: list[contracts.PulseSlot],+ *,+ complete: bool = False,+) -> contracts.Highlights:+ metrics = {metric.key: metric for metric in content.pulse}+ offered = {metric.key for source in progress.values() for metric in source.found.pulse}+ slots = {slot.key: slot for slot in previous}+ owners: dict[str, int] = {}+ order = {source.name: source.pulse_order for source in load_sources()}+ for name, source in progress.items():+ for metric in source.found.pulse:+ owners[metric.key] = order.get(name, 0)+ for slot in source.slots:+ slots[slot.key] = slot+ owners[slot.key] = order.get(name, 0)+ for metric in content.pulse:+ slots[metric.key] = contracts.PulseSlot(+ key=metric.key,+ name=metric.name,+ url=metric.url,+ interval=metric.interval,+ reason=metric.reason,+ status=PulseSlotStatus.READY,+ )+ pin_order = {pin.key: index for index, pin in enumerate(content.pins)}+ selected = [slot for slot in slots.values() if slot.key not in offered or slot.key in metrics]+ selected.sort(+ key=lambda slot: (+ slot.key not in pin_order,+ pin_order.get(slot.key, 0),+ slot.reason == HighlightMetricReason.POPULAR,+ owners.get(slot.key, 0),+ )+ )+ if complete:+ selected = [+ replace(slot, status=PulseSlotStatus.ERROR) if slot.status == PulseSlotStatus.PENDING else slot+ for slot in selected+ ]+ return replace(content, pulse_slots=selected)+++def publish_pulse(inputs: HighlightSourceInputs, found: HighlightFindings, slots: list[contracts.PulseSlot]) -> None:+ if inputs.highlights_id is None:+ return+ with transaction.atomic():+ row = (+ TodayHighlights.objects.for_team(inputs.team_id)+ .select_for_update()+ .filter(id=inputs.highlights_id, status=HighlightsStatus.COMPUTING)+ .first()+ )+ if row is None:+ return+ progress = _PROGRESS.validate_python(row.content.get("_pulse_sources", {}))+ previous = progress.get(inputs.source_name)+ if previous is not None:+ metrics = {metric.key: metric for metric in previous.found.pulse}+ metrics.update({metric.key: metric for metric in found.pulse})+ found = replace(found, pulse=tuple(metrics.values()))+ slots = list({slot.key: slot for slot in [*previous.slots, *slots]}.values())+ progress[inputs.source_name] = PulseSourceProgress(found=found, slots=slots)+ content = build_highlights(+ load_sources(),+ {name: source.found for name, source in progress.items()},+ Team.objects.get(id=inputs.team_id),+ row.local_day,+ )+ previous_slots = _HIGHLIGHTS.validate_python(row.content).pulse_slots if row.content else []+ content = _pulse_content(content, progress, previous_slots or [])+ row.content = _HIGHLIGHTS.dump_python(replace(content, status=HighlightsStatus.COMPUTING), mode="json")+ row.content["_pulse_sources"] = _PROGRESS.dump_python(progress, mode="json")+ row.save(update_fields=["content"])++def run_source(inputs: HighlightSourceInputs) -> HighlightSourceResult:source = next((source for source in load_sources() if source.name == inputs.source_name), None)context = Context(@@ -147,21 +276,36 @@ def run_source(inputs: HighlightSourceInputs) -> HighlightSourceResult:date.fromisoformat(inputs.local_day),None if inputs.scopes is None else frozenset(inputs.scopes),frozenset(inputs.pinned),+ publish_pulse=partial(publish_pulse, inputs) if inputs.highlights_id is not None else None,)if source is None or not context.allows(*source.scopes):return HighlightSourceResult(source_name=inputs.source_name, found=None, failed=False)with tags_context(product=Product.TODAY, feature=Feature.QUERY, team_id=context.team.id):found = context.guard(source.name, partial(source.find, context), HighlightFindings())+ if context.publish_pulse is not None:+ context.publish_pulse(found, [])return HighlightSourceResult(source_name=source.name, found=_FINDINGS.dump_json(found).decode(), failed=bool(context.failures))def finalize(inputs: FinalizeHighlightsInputs) -> None:- row = TodayHighlights.objects.for_team(inputs.team_id).filter(id=inputs.highlights_id).first()+ row = (+ TodayHighlights.objects.for_team(inputs.team_id)+ .select_for_update()+ .filter(id=inputs.highlights_id, status=HighlightsStatus.COMPUTING)+ .first()+ )if row is None:returnfound = {result.source_name: _FINDINGS.validate_json(result.found) for result in inputs.results if result.found}+ progress = _PROGRESS.validate_python(row.content.get("_pulse_sources", {}))+ for name, source in progress.items():+ final = found.get(name, source.found)+ metrics = {metric.key: metric for metric in source.found.pulse}+ metrics.update({metric.key: metric for metric in final.pulse})+ found[name] = replace(final, pulse=tuple(metrics.values()))highlights = build_highlights(load_sources(),found,@@ -170,6 +314,10 @@ def finalize(inputs: FinalizeHighlightsInputs) -> None:pins=[pin.key for pin in inputs.pins],pinned={metric.key: metric for metric in PINNED_METRICS.validate_json(inputs.pinned)},)+ if "_pulse_sources" in row.content:+ highlights = _pulse_content(+ highlights, progress, _HIGHLIGHTS.validate_python(row.content).pulse_slots or [], complete=True+ )failed = [result for result in inputs.results if result.failed]usable = bool(highlights.items or highlights.pulse)row.status = (diff --git a/products/today/backend/logic/today_highlights/key_metrics.py b/products/today/backend/logic/today_highlights/key_metrics.pyindex e3161085b077..2f78d57c9793 100644--- a/products/today/backend/logic/today_highlights/key_metrics.py+++ b/products/today/backend/logic/today_highlights/key_metrics.py@@ -1,11 +1,13 @@import json+from dataclasses import replace+from time import monotonic, sleepfrom urllib.parse import quote-from products.customer_analytics.backend.facade.api import get_key_metrics+from products.customer_analytics.backend.facade.api import get_key_metric_names, get_key_metricsfrom products.customer_analytics.backend.facade.contracts import KeyMetricfrom ...facade import contracts-from ...facade.enums import HighlightLabel, HighlightMetricReason, HighlightUnit+from ...facade.enums import HighlightLabel, HighlightMetricReason, HighlightUnit, PulseSlotStatusfrom ..highlight_rules import exactfrom ..highlight_trends import detectfrom ..highlight_types import Candidate, Context, DailyValueNotReady, Found, Source@@ -20,12 +22,36 @@ PULSE_DAYS = 28def find(context: Context) -> Found:+ slots = (+ [contracts.PulseSlot(key=key, name=name) for key, name in get_key_metric_names(team=context.team).items()]+ if context.publish_pulse+ else []+ )+ if context.publish_pulse is not None:+ context.publish_pulse(Found(), slots)metrics = get_key_metrics(team=context.team, user=context.user)+ if context.publish_pulse is not None:+ deadline = monotonic() + 20+ while metrics is None and monotonic() < deadline:+ sleep(min(1.0, max(0, deadline - monotonic())))+ metrics = get_key_metrics(team=context.team, user=context.user)+ if metrics is None:+ context.publish_pulse(Found(), [replace(slot, status=PulseSlotStatus.ERROR) for slot in slots])if metrics is None:raise DailyValueNotReady("key_metrics")shown = [metric for metric in metrics if metric.this_week or metric.last_week]candidates = [candidate for metric in shown if (candidate := _same_weekday(context, metric)) is not None]- return Found(items=tuple(candidates), pulse=tuple(tile(metric) for metric in shown))+ found = Found(items=tuple(candidates), pulse=tuple(tile(metric) for metric in shown))+ if context.publish_pulse is not None:+ keys = {metric.key for metric in found.pulse}+ context.publish_pulse(+ found,+ [+ replace(slot, status=PulseSlotStatus.READY if slot.key in keys else PulseSlotStatus.EMPTY)+ for slot in slots+ ],+ )+ return founddef tile(metric: KeyMetric) -> contracts.HighlightMetric:diff --git a/products/today/backend/logic/today_highlights/routine.py b/products/today/backend/logic/today_highlights/routine.pyindex a82af7ebdd53..bfe4af378285 100644--- a/products/today/backend/logic/today_highlights/routine.py+++ b/products/today/backend/logic/today_highlights/routine.py@@ -1,12 +1,14 @@from collections.abc import Collection, Iterator, Sequence+from dataclasses import replacefrom datetime import date, timedeltafrom functools import partial-from time import monotonic+from time import monotonic, sleepfrom django.db.models import Count, Maxfrom django.utils import timezonefrom posthog.dataclasses import frozen+from posthog.exceptions_capture import capture_exceptionfrom posthog.models import FileSystemViewLogfrom posthog.models.file_system.file_system_view_log import VIEW_DAYS_KEPT@@ -16,11 +18,16 @@ from products.dashboards.backend.facade.api import (dashboard_insight_tile,dashboard_insights,)-from products.product_analytics.backend.facade.api import get_insight_series-from products.product_analytics.backend.facade.contracts import InsightDashboardContext+from products.product_analytics.backend.facade.api import (+ get_insight_series,+ insight_query_complete,+ insight_series_supported,+ viewable_insight_definitions,+)+from products.product_analytics.backend.facade.contracts import InsightDashboardContext, InsightSeriesNotReadyfrom ...facade import contracts-from ...facade.enums import HighlightInterval, HighlightMetricReason, HighlightUnit+from ...facade.enums import HighlightInterval, HighlightMetricReason, HighlightUnit, PulseSlotStatusfrom ..highlight_rules import shortfrom ..highlight_series import pretty, usable_seriesfrom ..highlight_trends import complete, detect@@ -46,11 +53,132 @@ class ViewedItem:popular: bool = False+@frozen+class PulseCandidate:+ viewed: ViewedItem+ short_id: str+ dashboard: DashboardInsights | None = None+ tile: DashboardInsightTile | None = None++def find(context: Context) -> Found:pinned = context.pinned+ if context.publish_pulse is not None:+ return Found(pulse=tuple(_progressive_metrics(context, pinned, PULSE_CANDIDATES - len(pinned))))return Found(pulse=tuple(_frecent_metrics(context, pinned, PULSE_CANDIDATES - len(pinned))))+def _dashboard_context(candidate: PulseCandidate) -> InsightDashboardContext | None:+ if candidate.dashboard is None:+ return None+ return InsightDashboardContext(+ filters=candidate.dashboard.filters,+ tile_filters=candidate.tile.filters_overrides if candidate.tile else None,+ variables=candidate.dashboard.variables,+ )+++def _progressive_metrics(context: Context, excluded: Collection[str], budget: int) -> list[contracts.HighlightMetric]:+ if budget <= 0 or context.publish_pulse is None:+ return []+ deadline = monotonic() + BLOCKING_SECONDS+ candidates: list[PulseCandidate] = []+ for item in _candidates(context, _frecent_views(context)[:FRECENCY_CANDIDATES]):+ if item.kind == "insight":+ candidates.append(PulseCandidate(viewed=item, short_id=item.ref))+ elif item.ref.isdigit():+ dashboard = dashboard_insights(+ team=context.team, user=context.user, dashboard_id=int(item.ref), limit=DASHBOARD_TILES_TRIED+ )+ if dashboard is not None:+ candidates.extend(+ PulseCandidate(viewed=item, short_id=tile.insight_short_id, dashboard=dashboard, tile=tile)+ for tile in dashboard.tiles+ )+ definitions = viewable_insight_definitions(+ team=context.team, user=context.user, short_ids=[candidate.short_id for candidate in candidates]+ )+ selected: dict[str, PulseCandidate] = {}+ slots: dict[str, contracts.PulseSlot] = {}+ dashboards: set[int] = set()+ for candidate in candidates:+ key = f"insight:{candidate.short_id}"+ definition = definitions.get(candidate.short_id)+ if key in excluded or key in selected or definition is None:+ continue+ source = definition.query.get("source", definition.query)+ if not context.guard(+ f"eligible insight {candidate.short_id}",+ partial(insight_series_supported, query=definition.query, dashboard=_dashboard_context(candidate)),+ False,+ ):+ continue+ dashboard = candidate.dashboard+ if dashboard is not None and dashboard.dashboard_id in dashboards:+ continue+ interval = (dashboard.filters.get("interval") if dashboard else None) or source.get("interval", "day")+ selected[key] = candidate+ slots[key] = contracts.PulseSlot(+ key=key,+ name=definition.name or "Untitled insight",+ reason=HighlightMetricReason.POPULAR if candidate.viewed.popular else HighlightMetricReason.ROUTINE,+ url=f"/dashboard/{dashboard.dashboard_id}" if dashboard else f"/insights/{candidate.short_id}",+ interval=HighlightInterval(interval)+ if source.get("kind") == "TrendsQuery" and interval in HighlightInterval.values+ else None,+ )+ if dashboard is not None:+ dashboards.add(dashboard.dashboard_id)+ if len(selected) >= budget:+ break+ metrics: dict[str, contracts.HighlightMetric] = {}+ context.publish_pulse(Found(), list(slots.values()))+ pending = dict(selected)+ query_ids: dict[str, str] = {}+ while pending and monotonic() < deadline:+ for key, candidate in list(pending.items()):+ if monotonic() >= deadline:+ break+ dashboard = candidate.dashboard+ try:+ if key in query_ids and not insight_query_complete(team=context.team, query_id=query_ids[key]):+ continue+ metric = insight_metric(+ context,+ candidate.short_id,+ viewed=None if candidate.viewed.popular else candidate.viewed,+ dashboard=_dashboard_context(candidate),+ url=slots[key].url,+ dashboard_name=dashboard.name if dashboard else None,+ dashboard_id=dashboard.dashboard_id if dashboard else None,+ reason=HighlightMetricReason.POPULAR if candidate.viewed.popular else None,+ raise_on_pending=True,+ )+ except InsightSeriesNotReady as error:+ if error.query_id is None:+ raise ValueError("Pending insight has no query ID") from error+ query_ids[key] = error.query_id+ continue+ except Exception as error:+ context.failures.append(key)+ capture_exception(error, {"product": "today", "provider": "routine"})+ slots[key] = replace(slots[key], status=PulseSlotStatus.ERROR)+ else:+ slots[key] = replace(slots[key], status=PulseSlotStatus.READY if metric else PulseSlotStatus.EMPTY)+ if metric is not None:+ metrics[key] = metric+ del pending[key]+ context.publish_pulse(Found(pulse=tuple(metrics.values())), list(slots.values()))+ if pending:+ sleep(min(1.0, max(0, deadline - monotonic())))+ if pending:+ context.failures.extend(pending)+ for key in pending:+ slots[key] = replace(slots[key], status=PulseSlotStatus.ERROR)+ context.publish_pulse(Found(pulse=tuple(metrics.values())), list(slots.values()))+ return [metrics[key] for key in selected if key in metrics]++def frecency(view_days: Collection[date], today: date) -> float:ages = [(today - day).days for day in view_days]return sum(0.5 ** (age / FRECENCY_HALF_LIFE_DAYS) for age in ages if 0 <= age <= FRECENCY_LOOKBACK_DAYS)@@ -220,6 +348,7 @@ def insight_metric(reason: HighlightMetricReason | None = None,blocking: bool = False,wait_until: float | None = None,+ raise_on_pending: bool = False,) -> contracts.HighlightMetric | None:insight = get_insight_series(team=context.team,@@ -228,6 +357,7 @@ def insight_metric(dashboard=dashboard,blocking=blocking,wait_until=wait_until,+ raise_on_pending=raise_on_pending,)if insight is None:return Nonediff --git a/products/today/backend/presentation/serializers.py b/products/today/backend/presentation/serializers.pyindex cf117f4e8843..03a8337834c6 100644--- a/products/today/backend/presentation/serializers.py+++ b/products/today/backend/presentation/serializers.py@@ -23,6 +23,7 @@ from ..facade.contracts import (PullRequestLink,PulsePin,PulsePins,+ PulseSlot,RecordingTarget,ReportKeyClauses,ReportPage,@@ -42,6 +43,7 @@ from ..facade.enums import (HighlightUnit,ImpactNumberKey,KeyClauseRole,+ PulseSlotStatus,)@@ -439,10 +441,36 @@ class HighlightPinSerializer(DataclassSerializer):dataclass = HighlightPin+class PulseSlotSerializer(DataclassSerializer):+ reason = serializers.ChoiceField(+ choices=HighlightMetricReason.choices,+ allow_null=True,+ required=False,+ help_text="Why this number is selected, when known before its value arrives.",+ )+ key = serializers.CharField(help_text="Stable key of the selected number.")+ name = serializers.CharField(help_text="Title available before the number is computed.")+ status = serializers.ChoiceField(choices=PulseSlotStatus.choices, help_text="Loading state of this number.")+ url = serializers.CharField(allow_null=True, help_text="Path to the source, relative to the project.")+ interval = serializers.ChoiceField(+ choices=HighlightInterval.choices, allow_null=True, help_text="Measurement interval, when known."+ )++ class Meta:+ dataclass = PulseSlot+++class HighlightsQuerySerializer(serializers.Serializer):+ refresh = serializers.BooleanField(+ default=False,+ help_text="Retry failed sources when progressive loading is enabled. An active computation is reused.",+ )++class HighlightsSerializer(DataclassSerializer):status = serializers.ChoiceField(choices=HighlightsStatus.choices,- help_text="`ready` when the content is fresh. `computing` while new highlights are being computed: the content is the last ready highlights of the day, or empty, so poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.",+ help_text="`ready` when the content is fresh. `computing` while new highlights are being computed: the content includes available results and, when enabled, pending number details. Otherwise it is the last ready highlights of the day, or empty. Poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.",)generated_at = serializers.DateTimeField(allow_null=True, help_text="When the content was computed. Null when no highlights were computed yet today."@@ -452,6 +480,12 @@ class HighlightsSerializer(DataclassSerializer):help_text="Midnight at the end of `day` in the project's timezone, when the next day's highlights start. Load them again then.")pulse = HighlightMetricSerializer(many=True, help_text="Key metrics of the project from its settings.")+ pulse_slots = PulseSlotSerializer(+ many=True,+ allow_null=True,+ required=False,+ help_text="Selected numbers in stable display order, including unfinished numbers. Ready values are in pulse. Null for clients using complete results.",+ )items = HighlightItemSerializer(many=True,help_text="Up to 6 items, the highest scores across sources, in display order: by `priority`, then by `score`, highest first.",diff --git a/products/today/backend/presentation/views.py b/products/today/backend/presentation/views.pyindex c3c603c370b1..0b3f9f481bd0 100644--- a/products/today/backend/presentation/views.py+++ b/products/today/backend/presentation/views.py@@ -28,6 +28,7 @@ from .serializers import (ExcerptChoiceRequestSerializer,ExcerptChoiceSerializer,FigureMarksSerializer,+ HighlightsQuerySerializer,HighlightsSerializer,KeyClausesQuerySerializer,PulsePinsSerializer,@@ -104,6 +105,7 @@ class TodayViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):raise PermissionDenied()@validated_request(+ query_serializer=HighlightsQuerySerializer,responses={200: OpenApiResponse(response=HighlightsSerializer)},summary="Get today's highlights",description="The project's key metrics and up to 6 ranked items, each labeled risk or opportunity. Queries, rules and copy templates make every item; no AI is used. The highlights are computed in the background and stay fresh for 15 minutes per person, or 1 minute when a source failed. The call never waits: while they are computed, it returns `computing` with the last ready highlights of the day, so poll every 2 seconds until the status is `ready` or `failed`. 404 when the person does not have the new navigation. A scoped key needs query:read as well, because every highlight runs a query.",@@ -114,7 +116,12 @@ class TodayViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):if not api.is_enabled_for(user, self.team):raise NotFound()scopes = get_authenticator_scopes(request.successful_authenticator)- found = api.get_highlights(team=self.team, user=user, scopes=None if scopes is None else frozenset(scopes))+ found = api.get_highlights(+ team=self.team,+ user=user,+ scopes=None if scopes is None else frozenset(scopes),+ refresh=request.validated_query_data["refresh"],+ )return Response(HighlightsSerializer(found).data)@validated_request(diff --git a/products/today/backend/temporal/inputs.py b/products/today/backend/temporal/inputs.pyindex d2dfce46290b..280b3a1285f9 100644--- a/products/today/backend/temporal/inputs.py+++ b/products/today/backend/temporal/inputs.py@@ -20,6 +20,7 @@ class ComputeHighlightsInputs:scopes: list[str] | Nonesource_names: list[str]pins: list[PinnedNumber] = dataclasses.field(default_factory=list)+ progressive: bool = False@dataclasses.dataclass(frozen=True)@@ -30,6 +31,7 @@ class HighlightSourceInputs:scopes: list[str] | Nonesource_name: strpinned: list[str] = dataclasses.field(default_factory=list)+ highlights_id: str | None = None@dataclasses.dataclass(frozen=True)diff --git a/products/today/backend/temporal/workflows.py b/products/today/backend/temporal/workflows.pyindex 6257c82e2698..743dfdd1ea52 100644--- a/products/today/backend/temporal/workflows.py+++ b/products/today/backend/temporal/workflows.py@@ -61,6 +61,7 @@ async def _run_source(inputs: ComputeHighlightsInputs, source_name: str) -> Highscopes=inputs.scopes,source_name=source_name,pinned=[pin.key for pin in inputs.pins],+ highlights_id=inputs.highlights_id if inputs.progressive else None,),start_to_close_timeout=HIGHLIGHT_SOURCE_TIMEOUT,retry_policy=temporalio.common.RetryPolicy(maximum_attempts=2),diff --git a/products/today/backend/tests/test_highlight_insights.py b/products/today/backend/tests/test_highlight_insights.pyindex 1460fe294bf2..de5678a4b448 100644--- a/products/today/backend/tests/test_highlight_insights.py+++ b/products/today/backend/tests/test_highlight_insights.py@@ -1,4 +1,5 @@import re+from collections import Counterfrom datetime import UTC, date, datetime, time, timedeltafrom types import SimpleNamespacefrom typing import Any, cast@@ -15,7 +16,14 @@ from parameterized import parameterizedfrom posthog.models import FileSystemViewLog, Userfrom products.dashboards.backend.facade.api import DashboardInsights, DashboardInsightTile-from products.product_analytics.backend.facade.contracts import InsightDashboardContext, InsightSeries+from products.product_analytics.backend.facade.contracts import (+ InsightDashboardContext,+ InsightSeries,+ InsightSeriesNotReady,+ SavedInsightDefinition,+)+from products.today.backend.facade.contracts import HighlightFindings, PulseSlot+from products.today.backend.facade.enums import PulseSlotStatusfrom products.today.backend.logic.highlight_rules import human, shortfrom products.today.backend.logic.highlight_trends import BAD_METRIC, MONEY, UNCLEAR_LABEL, UNKNOWN_DIRECTIONfrom products.today.backend.logic.highlight_types import Context@@ -469,3 +477,125 @@ class TestRoutineInsights(BaseTest):("insight:once", "popular", None),],)+++class TestProgressiveInsights(SimpleTestCase):+ def test_ready_numbers_are_published_while_the_first_query_is_still_running(self) -> None:+ clock = [0.0]+ published: list[tuple[float, HighlightFindings, list[PulseSlot]]] = []+ ready_at = {"slow": 8.0, "fast": 1.0, "empty": 0.0, "timeout": 30.0}+ views = [routine.ViewedItem(kind="insight", ref=ref, days=frozenset({TODAY})) for ref in ready_at]+ definitions = {+ ref: SavedInsightDefinition(id=i, short_id=ref, name=ref, query={"kind": "TrendsQuery"})+ for i, ref in enumerate(ready_at)+ }+ queried: list[tuple[str, float]] = []++ def advance(seconds: float) -> None:+ clock[0] += seconds++ def series(**kwargs: Any) -> InsightSeries | None:+ ref = kwargs["short_id"]+ queried.append((ref, clock[0]))+ self.assertFalse(kwargs["blocking"])+ self.assertTrue(kwargs["raise_on_pending"])+ if clock[0] < ready_at[ref]:+ raise InsightSeriesNotReady(ref)+ return insight({"opened": days([10.0] * 56)}, short_id=ref, name=ref) if ref != "empty" else None++ context = cast(+ Context,+ SimpleNamespace(+ team=None,+ user=None,+ today=TODAY,+ failures=[],+ guard=lambda name, compute, fallback: compute(),+ publish_pulse=lambda found, slots: published.append((clock[0], found, slots)),+ ),+ )+ with (+ patch.object(routine, "_frecent_views", return_value=views),+ patch.object(routine, "_candidates", side_effect=lambda context, own: iter(own)),+ patch.object(routine, "viewable_insight_definitions", return_value=definitions),+ patch.object(routine, "get_insight_series", side_effect=series),+ patch.object(+ routine, "insight_query_complete", side_effect=lambda **kwargs: clock[0] >= ready_at[kwargs["query_id"]]+ ),+ patch.object(routine, "monotonic", side_effect=lambda: clock[0]),+ patch.object(routine, "sleep", side_effect=advance),+ ):+ found = routine._progressive_metrics(context, frozenset(), 6)++ self.assertEqual(published[0][0], 0)+ self.assertTrue(all(slot.status == PulseSlotStatus.PENDING for slot in published[0][2]))+ first_number = next((at, result) for at, result, _ in published if result.pulse)+ self.assertEqual(first_number[0], 1.0)+ self.assertEqual(first_number[1].pulse[0].key, "insight:fast")+ self.assertEqual([metric.key for metric in found], ["insight:slow", "insight:fast"])+ self.assertEqual(clock[0], 20.0)+ self.assertEqual({ref for ref, at in queried if at == 0}, set(ready_at))+ self.assertEqual(+ [slot.status for slot in published[-1][2]],+ [PulseSlotStatus.READY, PulseSlotStatus.READY, PulseSlotStatus.EMPTY, PulseSlotStatus.ERROR],+ )+ self.assertEqual(context.failures, ["insight:timeout"])+ self.assertEqual(Counter(ref for ref, _ in queried), {"slow": 2, "fast": 2, "empty": 1, "timeout": 1})+ self.assertTrue(+ all([slot.key for slot in slots] == [slot.key for slot in published[0][2]] for _, _, slots in published)+ )++ def test_a_dashboard_skips_an_unsupported_tile_before_selecting_a_loading_card(self) -> None:+ viewed = routine.ViewedItem(kind="dashboard", ref="7", days=frozenset({TODAY}))+ dashboard = DashboardInsights(+ dashboard_id=7,+ name="Growth",+ filters={},+ variables=None,+ tiles=(+ DashboardInsightTile(insight_short_id="number", filters_overrides=None),+ DashboardInsightTile(insight_short_id="chart", filters_overrides=None),+ ),+ )+ definitions = {+ ref: SavedInsightDefinition(+ id=i,+ short_id=ref,+ name=ref,+ query={+ "kind": "InsightVizNode",+ "source": {+ "kind": "TrendsQuery",+ "series": [{"kind": "EventsNode", "event": "opened"}],+ "trendsFilter": {"display": display},+ },+ },+ )+ for i, (ref, display) in enumerate([("number", "BoldNumber"), ("chart", "ActionsLineGraph")])+ }+ published: list[list[PulseSlot]] = []+ context = cast(+ Context,+ SimpleNamespace(+ team=None,+ user=None,+ today=TODAY,+ failures=[],+ guard=lambda name, compute, fallback: compute(),+ publish_pulse=lambda found, slots: published.append(slots),+ ),+ )+ with (+ patch.object(routine, "_frecent_views", return_value=[viewed]),+ patch.object(routine, "_candidates", side_effect=lambda context, own: iter(own)),+ patch.object(routine, "dashboard_insights", return_value=dashboard),+ patch.object(routine, "viewable_insight_definitions", return_value=definitions),+ patch.object(+ routine, "get_insight_series", return_value=insight({"opened": days([10.0] * 56)}, short_id="chart")+ ) as query,+ ):+ found = routine._progressive_metrics(context, frozenset(), 6)+ self.assertEqual([slot.key for slot in published[0]], ["insight:chart"])+ self.assertEqual([metric.key for metric in found], ["insight:chart"])+ query.assert_called_once()+ self.assertEqual(query.call_args.kwargs["short_id"], "chart")diff --git a/products/today/backend/tests/test_highlight_key_metrics.py b/products/today/backend/tests/test_highlight_key_metrics.pyindex ec92961b04a2..f73034947a99 100644--- a/products/today/backend/tests/test_highlight_key_metrics.py+++ b/products/today/backend/tests/test_highlight_key_metrics.py@@ -62,6 +62,7 @@ class TestKeyMetrics(SimpleTestCase):team=SimpleNamespace(id=1),user=SimpleNamespace(id=2),today=TODAY,+ publish_pulse=None,),)registered = next(s for s in load_sources() if s.name == "key_metrics")@@ -89,7 +90,10 @@ class TestKeyMetrics(SimpleTestCase):)def test_a_pending_calculation_asks_for_a_quick_retry(self) -> None:- context = cast(Context, SimpleNamespace(team=SimpleNamespace(id=1), user=SimpleNamespace(id=2), today=TODAY))+ context = cast(+ Context,+ SimpleNamespace(team=SimpleNamespace(id=1), user=SimpleNamespace(id=2), today=TODAY, publish_pulse=None),+ )registered = next(s for s in load_sources() if s.name == "key_metrics")with patch.object(key_metrics, "get_key_metrics", return_value=None), self.assertRaises(DailyValueNotReady):diff --git a/products/today/backend/tests/test_highlights.py b/products/today/backend/tests/test_highlights.pyindex 2f5f5296d20c..96b1729e1f8e 100644--- a/products/today/backend/tests/test_highlights.py+++ b/products/today/backend/tests/test_highlights.py@@ -1,15 +1,21 @@+from collections.abc import Callable+from concurrent.futures import ThreadPoolExecutorfrom dataclasses import replacefrom datetime import UTC, date, datetime, timedeltafrom functools import partial+from queue import Queue+from time import monotonic, sleepfrom types import SimpleNamespacefrom typing import Anyfrom zoneinfo import ZoneInfo+import pytestimport time_machinefrom posthog.test.base import BaseTestfrom unittest.mock import AsyncMock, MagicMock, patchfrom django.core.cache import cache+from django.db import close_old_connections, connection, connections, transactionfrom django.test import SimpleTestCasefrom django.utils import timezone@@ -34,6 +40,7 @@ from products.today.backend.facade.enums import (HighlightPriority,HighlightsStatus,HighlightUnit,+ PulseSlotStatus,)from products.today.backend.logic import highlight_registry, highlight_types, highlightsfrom products.today.backend.logic.highlight_types import Context@@ -495,3 +502,224 @@ class TestLoadSources(SimpleTestCase):registry = highlight_registry.load_registry.__wrapped__()self.assertEqual(dict(registry.products), {"flags": "feature_flags", "routine": "today"})+++@time_machine.travel(datetime(2026, 10, 13, 12, 0, tzinfo=UTC), tick=False)+class TestProgressiveHighlights(BaseTest):+ def test_partial_values_survive_source_retries_and_final_failure(self) -> None:+ row = TodayHighlights.objects.for_team(self.team.id).create(+ team_id=self.team.id, user_id=self.user.id, local_day=DAY, key="progress"+ )+ inputs = HighlightSourceInputs(+ team_id=self.team.id,+ user_id=self.user.id,+ local_day=DAY.isoformat(),+ scopes=None,+ source_name="routine",+ highlights_id=str(row.id),+ )+ slots = [contracts.PulseSlot(key="fast", name="Fast"), contracts.PulseSlot(key="slow", name="Slow")]+ highlights.publish_pulse(inputs, HighlightFindings(), slots)+ highlights.publish_pulse(inputs, HighlightFindings(pulse=(metric("fast"),)), slots)+ highlights.publish_pulse(inputs, HighlightFindings(), slots)+ row.refresh_from_db()+ partial = HIGHLIGHTS.validate_python(row.content)+ self.assertEqual(partial.status, HighlightsStatus.COMPUTING)+ self.assertEqual([m.key for m in partial.pulse], ["fast"])+ self.assertEqual(+ [s.status for s in partial.pulse_slots or []], [PulseSlotStatus.READY, PulseSlotStatus.PENDING]+ )+ highlights.finalize(+ FinalizeHighlightsInputs(+ team_id=self.team.id,+ highlights_id=str(row.id),+ results=[HighlightSourceResult(source_name="routine", found=None, failed=True)],+ )+ )+ row.refresh_from_db()+ final = HIGHLIGHTS.validate_python(row.content)+ self.assertEqual(row.status, HighlightsStatus.READY)+ self.assertTrue(row.failed_sources)+ self.assertNotIn("_pulse_sources", row.content)+ self.assertEqual([m.key for m in final.pulse], ["fast"])+ self.assertEqual([s.status for s in final.pulse_slots or []], [PulseSlotStatus.READY, PulseSlotStatus.ERROR])+ before = row.content+ highlights.finalize(+ FinalizeHighlightsInputs(+ team_id=self.team.id,+ highlights_id=str(row.id),+ results=[HighlightSourceResult(source_name="routine", found=None, failed=True)],+ )+ )+ row.refresh_from_db()+ self.assertEqual(row.content, before)+ highlights.publish_pulse(inputs, HighlightFindings(pulse=(metric("slow"),)), slots)+ row.refresh_from_db()+ self.assertEqual(row.content, before)++ @parameterized.expand([5, 6])+ def test_progress_keeps_source_ranking_limit_and_dates_after_failure(self, routine_count: int) -> None:+ row = TodayHighlights.objects.for_team(self.team.id).create(+ team_id=self.team.id, user_id=self.user.id, local_day=DAY, key="ranked"+ )+ inputs = HighlightSourceInputs(+ team_id=self.team.id,+ user_id=self.user.id,+ local_day=DAY.isoformat(),+ scopes=None,+ source_name="key_metrics",+ highlights_id=str(row.id),+ )+ configured = [replace(metric(f"key:{i}"), reason=HighlightMetricReason.KEY_METRIC) for i in range(4)]+ routine = [replace(metric(f"insight:{i}"), reason=HighlightMetricReason.ROUTINE) for i in range(routine_count)]+ popular = replace(metric("popular"), reason=HighlightMetricReason.POPULAR)+ highlights.publish_pulse(inputs, HighlightFindings(pulse=tuple(configured)), [])+ routine_inputs = replace(inputs, source_name="routine")+ slots = [contracts.PulseSlot(key=m.key, name=m.name, reason=m.reason) for m in [*routine, popular]]+ highlights.publish_pulse(routine_inputs, HighlightFindings(), slots)+ row.refresh_from_db()+ self.assertEqual(+ [slot.key for slot in HIGHLIGHTS.validate_python(row.content).pulse_slots or []][:routine_count],+ [m.key for m in routine],+ )+ highlights.publish_pulse(routine_inputs, HighlightFindings(pulse=(*routine, popular)), slots)+ highlights.finalize(+ FinalizeHighlightsInputs(+ team_id=self.team.id,+ highlights_id=str(row.id),+ results=[+ HighlightSourceResult(source_name="routine", found=None, failed=True),+ HighlightSourceResult(source_name="key_metrics", found=None, failed=True),+ ],+ )+ )+ row.refresh_from_db()+ final = HIGHLIGHTS.validate_python(row.content)+ expected = [m.key for m in [*routine, *configured]][:6]+ self.assertEqual([m.key for m in final.pulse], expected)+ self.assertEqual([slot.key for slot in final.pulse_slots or []], expected)+ self.assertTrue(all(m.period_starts == [DAY - timedelta(days=1)] for m in final.pulse))++ def test_retry_reuses_active_workflow_and_keeps_ready_numbers(self) -> None:+ cache.clear()+ connect = MagicMock()+ connect.return_value.start_workflow = AsyncMock()+ with (+ patch.object(highlights, "progressive_pulse_enabled", return_value=True),+ patch.object(highlights, "sync_connect", connect),+ ):+ highlights.get_highlights(team=self.team, user=self.user, scopes=None)+ rows = TodayHighlights.objects.for_team(self.team.id)+ first = rows.get()+ inputs = HighlightSourceInputs(+ team_id=self.team.id,+ user_id=self.user.id,+ local_day=DAY.isoformat(),+ scopes=None,+ source_name="routine",+ highlights_id=str(first.id),+ )+ highlights.publish_pulse(+ inputs,+ HighlightFindings(pulse=(metric("fast"),)),+ [contracts.PulseSlot(key="fast", name="Fast"), contracts.PulseSlot(key="slow", name="Slow")],+ )+ highlights.finalize(+ FinalizeHighlightsInputs(+ team_id=self.team.id,+ highlights_id=str(first.id),+ results=[HighlightSourceResult(source_name="routine", found=None, failed=True)],+ )+ )+ rows.filter(id=first.id).update(created_at=timezone.now() - timedelta(seconds=1))+ ready = highlights.get_highlights(team=self.team, user=self.user, scopes=None, refresh=True)+ self.assertEqual(ready.status, HighlightsStatus.COMPUTING)+ self.assertEqual([m.key for m in ready.pulse], ["fast"])+ new = rows.exclude(id=first.id).get()+ highlights.publish_pulse(+ replace(inputs, highlights_id=str(new.id)),+ HighlightFindings(),+ [contracts.PulseSlot(key="fast", name="Fast"), contracts.PulseSlot(key="slow", name="Slow")],+ )+ pending = highlights.get_highlights(team=self.team, user=self.user, scopes=None, refresh=True)+ self.assertEqual([m.key for m in pending.pulse], ["fast"])+ self.assertEqual(+ [s.status for s in pending.pulse_slots or []], [PulseSlotStatus.READY, PulseSlotStatus.PENDING]+ )+ self.assertEqual(connect.return_value.start_workflow.call_count, 2)+ self.assertTrue(connect.return_value.start_workflow.call_args.args[1].progressive)+ retry_at = timezone.now() + timedelta(seconds=highlights.FAILED_CACHE_SECONDS + 1)+ with time_machine.travel(timezone.now() + timedelta(seconds=30), tick=False):+ highlights.finalize(+ FinalizeHighlightsInputs(+ team_id=self.team.id,+ highlights_id=str(new.id),+ results=[HighlightSourceResult(source_name="routine", found=None, failed=True)],+ )+ )+ for _ in range(3):+ limited = highlights.get_highlights(team=self.team, user=self.user, scopes=None, refresh=True)+ self.assertEqual(limited.status, HighlightsStatus.FAILED)+ self.assertEqual(rows.count(), 2)+ self.assertEqual(connect.return_value.start_workflow.call_count, 2)+ with time_machine.travel(retry_at, tick=False):+ retried = highlights.get_highlights(team=self.team, user=self.user, scopes=None, refresh=True)+ self.assertEqual(retried.status, HighlightsStatus.COMPUTING)+ self.assertEqual(rows.count(), 3)+ self.assertEqual(connect.return_value.start_workflow.call_count, 3)++[email protected]_db(transaction=True)+def test_finalization_keeps_a_concurrent_publication(team: Team, user: User) -> None:+ row = TodayHighlights.objects.for_team(team.id).create(+ team_id=team.id, user_id=user.id, local_day=DAY, key="concurrent"+ )+ source = HighlightSourceInputs(+ team_id=team.id,+ user_id=user.id,+ local_day=DAY.isoformat(),+ scopes=None,+ source_name="routine",+ highlights_id=str(row.id),+ )+ final = FinalizeHighlightsInputs(+ team_id=team.id,+ highlights_id=str(row.id),+ results=[HighlightSourceResult(source_name="routine", found=None, failed=True)],+ )+ pids: Queue[int] = Queue()++ def run(action: Callable[[], None]) -> None:+ close_old_connections()+ try:+ with connection.cursor() as cursor:+ cursor.execute("SELECT pg_backend_pid()")+ pids.put(cursor.fetchone()[0])+ action()+ finally:+ connections.close_all()++ def wait_for_blocked_connection(pid: int) -> None:+ deadline = monotonic() + 10+ while monotonic() < deadline:+ with connection.cursor() as cursor:+ cursor.execute("SELECT cardinality(pg_blocking_pids(%s))", [pid])+ if cursor.fetchone()[0] > 0:+ return+ sleep(0.01)+ pytest.fail("The concurrent database operation did not wait for the row lock")++ with ThreadPoolExecutor(max_workers=2) as pool:+ with transaction.atomic():+ TodayHighlights.objects.for_team(team.id).select_for_update().get(id=row.id)+ publish = pool.submit(+ run, partial(highlights.publish_pulse, source, HighlightFindings(pulse=(metric("fast"),)), [])+ )+ wait_for_blocked_connection(pids.get(timeout=10))+ finish = pool.submit(run, partial(highlights.finalize, final))+ wait_for_blocked_connection(pids.get(timeout=10))+ publish.result(timeout=10)+ finish.result(timeout=10)+ row.refresh_from_db()+ assert row.status == HighlightsStatus.READY+ assert [metric.key for metric in HIGHLIGHTS.validate_python(row.content).pulse] == ["fast"]diff --git a/products/today/backend/tests/test_workflows.py b/products/today/backend/tests/test_workflows.pyindex 53ca8dc3cb39..8357785cdcfc 100644--- a/products/today/backend/tests/test_workflows.py+++ b/products/today/backend/tests/test_workflows.py@@ -1,9 +1,14 @@import uuid+import asynciofrom collections.abc import AsyncIterator+from threading import Event+from typing import castimport pytest+from unittest.mock import patchfrom django.conf import settings+from django.utils import timezoneimport pytest_asynciofrom temporalio import activity@@ -11,6 +16,14 @@ from temporalio.exceptions import ApplicationErrorfrom temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Worker+from posthog.models import Team, User+from posthog.sync import database_sync_to_async++from products.today.backend.facade.contracts import HighlightContext, HighlightFindings, HighlightSource, PulseSlot+from products.today.backend.logic import highlights+from products.today.backend.logic.highlight_types import Context+from products.today.backend.models import TodayHighlights+from products.today.backend.temporal.activities import finalize_highlights_activity, run_highlight_source_activityfrom products.today.backend.temporal.inputs import (HIGHLIGHTS_WORKFLOW_NAME,ComputeHighlightsInputs,@@ -22,6 +35,7 @@ from products.today.backend.temporal.inputs import (ResolvePinsResult,)from products.today.backend.temporal.workflows import ComputeTodayHighlightsWorkflow+from products.today.backend.tests.test_highlights import HIGHLIGHTS, metric@pytest_asyncio.fixture(scope="module")@@ -34,6 +48,7 @@ PINS = [PinnedNumber("insight:a"), PinnedNumber("insight:b", dashboard_id=7)]@pytest.mark.asyncio[email protected]("progressive", [False, True])@pytest.mark.parametrize("pins,pins_fail,asked,pinned",[@@ -43,13 +58,19 @@ PINS = [PinnedNumber("insight:a"), PinnedNumber("insight:b", dashboard_id=7)]],)async def test_sources_and_pins_are_resolved_once_and_finalized_together(- environment: WorkflowEnvironment, pins: list[PinnedNumber], pins_fail: bool, asked: list[list[str]], pinned: str+ environment: WorkflowEnvironment,+ progressive: bool,+ pins: list[PinnedNumber],+ pins_fail: bool,+ asked: list[list[str]],+ pinned: str,) -> None:finalized: list[FinalizeHighlightsInputs] = []resolved: list[list[str]] = []@activity.defn(name="run_highlight_source_activity")async def run_source(inputs: HighlightSourceInputs) -> HighlightSourceResult:+ assert inputs.highlights_id == ("h-1" if progressive else None)if inputs.source_name == "broken":raise ApplicationError("The source query failed.")return HighlightSourceResult(source_name=inputs.source_name, found="{}", failed=False)@@ -82,6 +103,7 @@ async def test_sources_and_pins_are_resolved_once_and_finalized_together(scopes=None,source_names=["alerts", "broken"],pins=pins,+ progressive=progressive,),id=str(uuid.uuid4()),task_queue=settings.GENERAL_PURPOSE_TASK_QUEUE,@@ -100,3 +122,67 @@ async def test_sources_and_pins_are_resolved_once_and_finalized_together()],)++[email protected]_db(transaction=True)+async def test_real_activity_publishes_before_the_workflow_finishes(+ environment: WorkflowEnvironment, team: Team, user: User+) -> None:+ published = Event()+ finish = Event()+ day = timezone.now().astimezone(team.timezone_info).date()++ def create_row() -> TodayHighlights:+ return TodayHighlights.objects.for_team(team.id).create(+ team_id=team.id, user_id=user.id, local_day=day, key="progress-test"+ )++ row = await database_sync_to_async(create_row)()++ def find(context: HighlightContext) -> HighlightFindings:+ publisher = cast(Context, context).publish_pulse+ assert publisher is not None+ publisher(+ HighlightFindings(pulse=(metric("fast"),)),+ [PulseSlot(key="fast", name="Fast"), PulseSlot(key="slow", name="Slow")],+ )+ published.set()+ assert finish.wait(15)+ return HighlightFindings(pulse=(metric("fast"), metric("slow")))++ with patch.object(highlights, "load_sources", return_value=(HighlightSource(name="routine", find=find),)):+ async with Worker(+ environment.client,+ task_queue=settings.GENERAL_PURPOSE_TASK_QUEUE,+ workflows=[ComputeTodayHighlightsWorkflow],+ activities=[run_highlight_source_activity, finalize_highlights_activity],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ handle = await environment.client.start_workflow(+ HIGHLIGHTS_WORKFLOW_NAME,+ ComputeHighlightsInputs(+ team_id=team.id,+ user_id=user.id,+ highlights_id=str(row.id),+ local_day=day.isoformat(),+ scopes=None,+ source_names=["routine"],+ progressive=True,+ ),+ id=str(uuid.uuid4()),+ task_queue=settings.GENERAL_PURPOSE_TASK_QUEUE,+ )+ try:+ assert await asyncio.to_thread(published.wait, 15)+ await database_sync_to_async(row.refresh_from_db)()+ content = HIGHLIGHTS.validate_python(row.content)+ assert row.status == "computing"+ assert [m.key for m in content.pulse] == ["fast"]+ assert [slot.status for slot in content.pulse_slots or []] == ["ready", "pending"]+ finally:+ finish.set()+ await handle.result()+ await database_sync_to_async(row.refresh_from_db)()+ assert row.status == "ready"+ assert [m.key for m in HIGHLIGHTS.validate_python(row.content).pulse] == ["fast", "slow"]diff --git a/products/today/frontend/generated/api.schemas.ts b/products/today/frontend/generated/api.schemas.tsindex 76bd8aa5798e..80e8b3211460 100644--- a/products/today/frontend/generated/api.schemas.ts+++ b/products/today/frontend/generated/api.schemas.ts@@ -191,6 +191,53 @@ export interface HighlightMetricApi {movers: HighlightMoverApi[]}+/**+ * * `pending` - Loading+ * * `ready` - Ready+ * * `empty` - No data+ * * `error` - Could not load+ */+export type PulseSlotStatusEnumApi = (typeof PulseSlotStatusEnumApi)[keyof typeof PulseSlotStatusEnumApi]++export const PulseSlotStatusEnumApi = {+ Pending: 'pending',+ Ready: 'ready',+ Empty: 'empty',+ Error: 'error',+} as const++export interface PulseSlotApi {+ /** Why this number is selected, when known before its value arrives.+ *+ * * `routine` - You open this often+ * * `dashboard` - On a dashboard you open often+ * * `key_metric` - Project key metric+ * * `popular` - Opened often in this project */+ reason?: HighlightMetricReasonEnumApi | null+ /** Stable key of the selected number. */+ key: string+ /** Title available before the number is computed. */+ name: string+ /** Loading state of this number.+ *+ * * `pending` - Loading+ * * `ready` - Ready+ * * `empty` - No data+ * * `error` - Could not load */+ status: PulseSlotStatusEnumApi+ /**+ * Path to the source, relative to the project.+ * @nullable+ */+ url: string | null+ /** Measurement interval, when known.+ *+ * * `day` - Day+ * * `week` - Week+ * * `month` - Month */+ interval: CalendarUnitEnumApi | null+}+/*** * `risk` - Risk* * `opportunity` - Opportunity@@ -279,7 +326,7 @@ export interface HighlightPinApi {}export interface HighlightsApi {- /** `ready` when the content is fresh. `computing` while new highlights are being computed: the content is the last ready highlights of the day, or empty, so poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.+ /** `ready` when the content is fresh. `computing` while new highlights are being computed: the content includes available results and, when enabled, pending number details. Otherwise it is the last ready highlights of the day, or empty. Poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.** * `computing` - Computing* * `ready` - Ready@@ -296,6 +343,11 @@ export interface HighlightsApi {refresh_at: string/** Key metrics of the project from its settings. */pulse: HighlightMetricApi[]+ /**+ * Selected numbers in stable display order, including unfinished numbers. Ready values are in pulse. Null for clients using complete results.+ * @nullable+ */+ pulse_slots?: PulseSlotApi[] | null/** Up to 6 items, the highest scores across sources, in display order: by `priority`, then by `score`, highest first. */items: HighlightItemApi[]/** The numbers the person keeps, in their order, with the state of each one. Pinned numbers lead `pulse`. */@@ -601,6 +653,13 @@ export interface ReportPageApi {last_seen: string | null}+export type TodayHighlightsRetrieveParams = {+ /**+ * Retry failed sources when progressive loading is enabled. An active computation is reused.+ */+ refresh?: boolean+}+export type TodayReportsKeyClausesRetrieveParams = {/*** Whether to mark the impact sentence. Pass false when the page shows an impact number instead.diff --git a/products/today/frontend/generated/api.ts b/products/today/frontend/generated/api.tsindex 50cf87c02150..fe08c48293ce 100644--- a/products/today/frontend/generated/api.ts+++ b/products/today/frontend/generated/api.ts@@ -16,6 +16,7 @@ import type {PulsePinsApi,ReportKeyClausesApi,ReportPageApi,+ TodayHighlightsRetrieveParams,TodayReportsKeyClausesRetrieveParams,} from './api.schemas'@@ -40,16 +41,32 @@ export const todayExcerptChoiceCreate = async (})}-export const getTodayHighlightsRetrieveUrl = (projectId: string) => {- return `/api/projects/${projectId}/today/highlights/`+export const getTodayHighlightsRetrieveUrl = (projectId: string, params?: TodayHighlightsRetrieveParams) => {+ const normalizedParams = new URLSearchParams()++ Object.entries(params || {}).forEach(([key, value]) => {+ if (value !== undefined) {+ normalizedParams.append(key, value === null ? 'null' : String(value))+ }+ })++ const stringifiedParams = normalizedParams.toString()++ return stringifiedParams.length > 0+ ? `/api/projects/${projectId}/today/highlights/?${stringifiedParams}`+ : `/api/projects/${projectId}/today/highlights/`}/*** The project's key metrics and up to 6 ranked items, each labeled risk or opportunity. Queries, rules and copy templates make every item; no AI is used. The highlights are computed in the background and stay fresh for 15 minutes per person, or 1 minute when a source failed. The call never waits: while they are computed, it returns `computing` with the last ready highlights of the day, so poll every 2 seconds until the status is `ready` or `failed`. 404 when the person does not have the new navigation. A scoped key needs query:read as well, because every highlight runs a query.* @summary Get today's highlights*/-export const todayHighlightsRetrieve = async (projectId: string, options?: RequestInit): Promise<HighlightsApi> => {- return apiMutator<HighlightsApi>(getTodayHighlightsRetrieveUrl(projectId), {+export const todayHighlightsRetrieve = async (+ projectId: string,+ params?: TodayHighlightsRetrieveParams,+ options?: RequestInit+): Promise<HighlightsApi> => {+ return apiMutator<HighlightsApi>(getTodayHighlightsRetrieveUrl(projectId, params), {...options,method: 'GET',})diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex cef24c91c136..9a45d0ff8182 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -59708,8 +59708,56 @@ export namespace Schemas {Failed: 'failed',} as const;+ /**+ * * `pending` - Loading+ * * `ready` - Ready+ * * `empty` - No data+ * * `error` - Could not load+ */+ export type PulseSlotStatusEnum = typeof PulseSlotStatusEnum[keyof typeof PulseSlotStatusEnum];+++ export const PulseSlotStatusEnum = {+ Pending: 'pending',+ Ready: 'ready',+ Empty: 'empty',+ Error: 'error',+ } as const;++ export interface PulseSlot {+ /** Why this number is selected, when known before its value arrives.+ *+ * * `routine` - You open this often+ * * `dashboard` - On a dashboard you open often+ * * `key_metric` - Project key metric+ * * `popular` - Opened often in this project */+ reason?: HighlightMetricReasonEnum | null;+ /** Stable key of the selected number. */+ key: string;+ /** Title available before the number is computed. */+ name: string;+ /** Loading state of this number.+ *+ * * `pending` - Loading+ * * `ready` - Ready+ * * `empty` - No data+ * * `error` - Could not load */+ status: PulseSlotStatusEnum;+ /**+ * Path to the source, relative to the project.+ * @nullable+ */+ url: string | null;+ /** Measurement interval, when known.+ *+ * * `day` - Day+ * * `week` - Week+ * * `month` - Month */+ interval: CalendarUnitEnum | null;+ }+export interface Highlights {- /** `ready` when the content is fresh. `computing` while new highlights are being computed: the content is the last ready highlights of the day, or empty, so poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.+ /** `ready` when the content is fresh. `computing` while new highlights are being computed: the content includes available results and, when enabled, pending number details. Otherwise it is the last ready highlights of the day, or empty. Poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.** * `computing` - Computing* * `ready` - Ready@@ -59726,6 +59774,11 @@ export namespace Schemas {refresh_at: string;/** Key metrics of the project from its settings. */pulse: HighlightMetric[];+ /**+ * Selected numbers in stable display order, including unfinished numbers. Ready values are in pulse. Null for clients using complete results.+ * @nullable+ */+ pulse_slots?: PulseSlot[] | null;/** Up to 6 items, the highest scores across sources, in display order: by `priority`, then by `score`, highest first. */items: HighlightItem[];/** The numbers the person keeps, in their order, with the state of each one. Pinned numbers lead `pulse`. */@@ -144915,6 +144968,13 @@ export namespace Schemas {room: string;};+ export type TodayHighlightsRetrieveParams = {+ /**+ * Retry failed sources when progressive loading is enabled. An active computation is reused.+ */+ refresh?: boolean;+ };+export type TodayReportsKeyClausesRetrieveParams = {/*** Whether to mark the impact sentence. Pass false when the page shows an impact number instead.