Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(today): publish ready numbers before all sources finish (#127509)

Shy Alter100e7516aa00
diff --git a/products/customer_analytics/backend/facade/api.py b/products/customer_analytics/backend/facade/api.py
index 5706ad38a45e..bf9e3b57ec94 100644
--- a/products/customer_analytics/backend/facade/api.py
+++ b/products/customer_analytics/backend/facade/api.py
@@ -5952,3 +5952,7 @@ def list_announcement_channels(team_id: int) -> list[contracts.AnnouncementChann
def get_key_metrics(*, team: Team, user: "User") -> list[contracts.KeyMetric] | None:
"""The key metrics, or None while a background calculation is still running."""
return _key_metrics_logic.key_metrics(team, user)
+
+
+def get_key_metric_names(*, team: Team) -> dict[str, str]:
+ return _key_metrics_logic.key_metric_names(team)
diff --git a/products/customer_analytics/backend/logic/key_metrics.py b/products/customer_analytics/backend/logic/key_metrics.py
index 8325098d37a0..a8f3bc0d1a4e 100644
--- a/products/customer_analytics/backend/logic/key_metrics.py
+++ b/products/customer_analytics/backend/logic/key_metrics.py
@@ -56,6 +56,11 @@ def _action_ids(nodes: dict[str, dict[str, Any]]) -> list[int]:
return [int(node["id"]) for node in nodes.values() if node["kind"] == "ActionsNode"]
+def key_metric_names(team: Team) -> dict[str, str]:
+ nodes = _configured_nodes(team)
+ return {key: node.get("custom_name") or name for key, name in KEY_METRICS.items() if (node := nodes.get(key))}
+
+
def _anchored(filters: Any, team: Team, anchor: datetime) -> Any:
if isinstance(filters, list):
return [_anchored(item, team, anchor) for item in filters]
diff --git a/products/product_analytics/backend/facade/api.py b/products/product_analytics/backend/facade/api.py
index b7d3493328fd..bc104f537183 100644
--- a/products/product_analytics/backend/facade/api.py
+++ b/products/product_analytics/backend/facade/api.py
@@ -291,6 +291,40 @@ def save_saved_insight_query(
return None
+def viewable_insight_definitions(
+ *, team: Team, user: User, short_ids: Collection[str]
+) -> dict[str, SavedInsightDefinition]:
+ access_control = UserAccessControl(user=user, team=team, organization_id=str(team.organization_id))
+ insights = list(Insight.objects.filter(team__project_id=team.project_id, short_id__in=short_ids, deleted=False))
+ denied = set(
+ access_control.denied_object_ids(
+ "insight", ((str(insight.pk), insight.team_id, insight.created_by_id) for insight in insights), "viewer"
+ )
+ )
+ return {
+ insight.short_id: SavedInsightDefinition(
+ id=insight.pk,
+ short_id=insight.short_id,
+ name=insight.name or insight.derived_name,
+ query=insight.query or {},
+ )
+ for insight in insights
+ if str(insight.pk) not in denied
+ }
+
+
+def insight_query_complete(*, team: Team, query_id: str) -> bool:
+ from products.product_analytics.backend import insight_series
+
+ return insight_series.query_complete(team=team, query_id=query_id)
+
+
+def insight_series_supported(*, query: dict[str, Any], dashboard: InsightDashboardContext | None = None) -> bool:
+ from products.product_analytics.backend import insight_series
+
+ return insight_series.supports_series(query=query, dashboard=dashboard)
+
+
def get_insight_series(
*,
team: Team,
@@ -299,12 +333,15 @@ def get_insight_series(
dashboard: InsightDashboardContext | None = None,
blocking: bool = False,
wait_until: float | None = None,
+ raise_on_pending: bool = False,
) -> InsightSeries | None:
insight = Insight.objects.filter(team__project_id=team.project_id, short_id=short_id, deleted=False).first()
if insight is None:
return None
access_control = UserAccessControl(user=user, team=team, organization_id=str(team.organization_id))
- if not access_control.check_access_level_for_object(insight, "viewer"):
+ if access_control.denied_object_ids(
+ "insight", [(str(insight.pk), insight.team_id, insight.created_by_id)], "viewer"
+ ):
return None
from products.product_analytics.backend import ( # noqa: PLC0415 -- insight_series imports the query runner, which imports this facade
insight_series,
@@ -319,4 +356,5 @@ def get_insight_series(
dashboard=dashboard,
blocking=blocking,
wait_until=wait_until,
+ raise_on_pending=raise_on_pending,
)
diff --git a/products/product_analytics/backend/facade/contracts.py b/products/product_analytics/backend/facade/contracts.py
index 82977a4c7e9a..4a4d925d4592 100644
--- a/products/product_analytics/backend/facade/contracts.py
+++ b/products/product_analytics/backend/facade/contracts.py
@@ -73,6 +73,12 @@ class SavedInsightDefinition:
query: dict[str, Any]
+class InsightSeriesNotReady(Exception):
+ def __init__(self, query_id: str | None = None) -> None:
+ super().__init__(query_id)
+ self.query_id = query_id
+
+
@dataclass(frozen=True)
class InsightDataCheckFilter:
"""A rule in the shape the project's internal and test users filter stores it."""
diff --git a/products/product_analytics/backend/insight_series.py b/products/product_analytics/backend/insight_series.py
index 1a1ebe4afb8f..99b19cab466c 100644
--- a/products/product_analytics/backend/insight_series.py
+++ b/products/product_analytics/backend/insight_series.py
@@ -20,7 +20,11 @@ from posthog.models import Team, User
from posthog.schema_migrations.upgrade import upgrade
from posthog.week_start_day import WeekStartDay
-from products.product_analytics.backend.facade.contracts import InsightDashboardContext, InsightSeries
+from products.product_analytics.backend.facade.contracts import (
+ InsightDashboardContext,
+ InsightSeries,
+ InsightSeriesNotReady,
+)
ADDITIVE_MATH = {
"total",
@@ -47,6 +51,37 @@ EXECUTION_MODE = ExecutionMode.RECENT_CACHE_CALCULATE_ASYNC_IF_STALE
BLOCKING_EXECUTION_MODE = ExecutionMode.RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE
+def query_complete(*, team: Team, query_id: str) -> bool:
+ status = get_query_status(team_id=team.pk, query_id=query_id)
+ if status.error:
+ raise ValueError("Insight query failed")
+ return bool(status.complete)
+
+
+def _supports_series(query: dict[str, Any], filters: dict[str, Any]) -> bool:
+ source = query.get("source", query)
+ if query.get("kind") == "DataVisualizationNode" and source.get("kind") == "HogQLQuery":
+ settings = query.get("chartSettings") or {}
+ return bool(
+ query.get("display") in CHARTED_SQL_DISPLAYS
+ and (settings.get("xAxis") or {}).get("column")
+ and any(axis.get("column") for axis in settings.get("yAxis") or [])
+ )
+ if source.get("kind") != "TrendsQuery" or (source.get("trendsFilter") or {}).get("display") in UNCHARTED_DISPLAYS:
+ return False
+ interval = filters.get("interval") or source.get("interval") or "day"
+ if interval not in LOOKBACK:
+ interval = "day"
+ return interval != "day" or not (source.get("dateRange") or {}).get("daysOfWeek")
+
+
+def supports_series(*, query: dict[str, Any], dashboard: InsightDashboardContext | None = None) -> bool:
+ if dashboard is None:
+ return _supports_series(query, {})
+ effective = resolve_effective_dashboard_filters(query, dashboard.filters, dashboard.tile_filters)
+ return _supports_series(effective.query, effective.filters or {})
+
+
def _run(
team: Team,
user: User,
@@ -55,6 +90,7 @@ def _run(
variables: dict[str, Any] | None,
blocking: bool = False,
wait_until: float | None = None,
+ raise_on_pending: bool = False,
) -> dict[str, Any]:
from posthog.api.services.query import process_query_dict # noqa: PLC0415
@@ -67,6 +103,13 @@ def _run(
user=user,
)
result = response.model_dump() if isinstance(response, BaseModel) else response
+ status = result.get("query_status") or {}
+ if raise_on_pending and (result.get("error") or status.get("error")):
+ raise ValueError("Insight query failed")
+ if raise_on_pending and result.get("results") is None:
+ if not status.get("id"):
+ raise ValueError("Insight query returned no results or query status")
+ raise InsightSeriesNotReady(status["id"])
if blocking and wait_until is not None and result.get("results") is None:
status = result.get("query_status") or {}
while status and not status.get("complete") and not status.get("error"):
@@ -143,13 +186,12 @@ def _sql_series(
variables: dict[str, Any] | None,
blocking: bool = False,
wait_until: float | None = None,
+ raise_on_pending: bool = False,
) -> InsightSeries | None:
settings = query.get("chartSettings") or {}
x_column = (settings.get("xAxis") or {}).get("column")
y_axes = [axis for axis in settings.get("yAxis") or [] if axis.get("column")]
- if query.get("display") not in CHARTED_SQL_DISPLAYS or not x_column or not y_axes:
- return None
- response = _run(team, user, query["source"], filters, variables, blocking, wait_until)
+ response = _run(team, user, query["source"], filters, variables, blocking, wait_until, raise_on_pending)
if response.get("hasMore"):
return None
position = {column: index for index, column in enumerate(response.get("columns") or [])}
@@ -219,6 +261,7 @@ def insight_series(
dashboard: InsightDashboardContext | None,
blocking: bool = False,
wait_until: float | None = None,
+ raise_on_pending: bool = False,
) -> InsightSeries | None:
dashboard_filters: dict[str, Any] = {}
variables: dict[str, Any] | None = None
@@ -228,6 +271,8 @@ def insight_series(
dashboard_filters = effective.filters or {}
variables = dashboard.variables
filters = {key: value for key, value in dashboard_filters.items() if key not in DATE_FILTERS}
+ if not _supports_series(query, dashboard_filters):
+ return None
source = query.get("source", query)
if query.get("kind") == "DataVisualizationNode" and source.get("kind") == "HogQLQuery":
return _sql_series(
@@ -240,16 +285,13 @@ def insight_series(
variables=variables,
blocking=blocking,
wait_until=wait_until,
+ raise_on_pending=raise_on_pending,
)
trends = source.get("trendsFilter") or {}
- if source.get("kind") != "TrendsQuery" or trends.get("display") in UNCHARTED_DISPLAYS:
- return None
interval = dashboard_filters.get("interval") or source.get("interval") or "day"
if interval not in LOOKBACK:
interval = "day"
date_range = source.get("dateRange") or {}
- if interval == "day" and date_range.get("daysOfWeek"):
- return None
kept = {key: value for key, value in date_range.items() if key not in DATE_FILTERS}
bounded = {key: value for key, value in source.items() if key != "compareFilter"}
bounded.update(
@@ -257,7 +299,7 @@ def insight_series(
dateRange={**kept, "date_from": LOOKBACK[interval], "excludeIncompletePeriods": True},
trendsFilter={**{k: v for k, v in trends.items() if k != "smoothingIntervals"}, "metricShowChange": False},
)
- response = _run(team, user, bounded, filters, variables, blocking, wait_until)
+ response = _run(team, user, bounded, filters, variables, blocking, wait_until, raise_on_pending)
formula = bool(trends.get("formula") or trends.get("formulas") or trends.get("formulaNodes"))
several = len(source.get("series") or []) > 1
# `_run` executes the upgraded query, which stamps `kind` on series saved without one.
diff --git a/products/product_analytics/backend/tests/test_insight_series.py b/products/product_analytics/backend/tests/test_insight_series.py
index 552515131e61..2e36caaa0158 100644
--- a/products/product_analytics/backend/tests/test_insight_series.py
+++ b/products/product_analytics/backend/tests/test_insight_series.py
@@ -20,13 +20,55 @@ from posthog.week_start_day import WeekStartDay
from products.access_control.backend.facade.testing import create_access_control
from products.actions.backend.models.action import Action
from products.product_analytics.backend import insight_series
-from products.product_analytics.backend.facade.api import get_insight_series
-from products.product_analytics.backend.facade.contracts import InsightDashboardContext
+from products.product_analytics.backend.facade.api import get_insight_series, viewable_insight_definitions
+from products.product_analytics.backend.facade.contracts import InsightDashboardContext, InsightSeriesNotReady
from products.product_analytics.backend.insight_series import _visible
from products.product_analytics.backend.models.insight import Insight
class TestBoundedInsightQuery(SimpleTestCase):
+ @parameterized.expand([(False, False), (True, False), (True, True)])
+ def test_query_status_is_scoped_and_does_not_prepare_the_query(self, complete: bool, error: bool) -> None:
+ with (
+ patch.object(
+ insight_series,
+ "get_query_status",
+ return_value=QueryStatus(id="q", team_id=7, complete=complete, error=error),
+ ) as status,
+ patch("posthog.api.services.query.process_query_dict") as prepare,
+ ):
+ if error:
+ with self.assertRaises(ValueError):
+ insight_series.query_complete(team=cast(Team, SimpleNamespace(pk=7)), query_id="q")
+ else:
+ self.assertEqual(
+ insight_series.query_complete(team=cast(Team, SimpleNamespace(pk=7)), query_id="q"), complete
+ )
+ status.assert_called_once_with(team_id=7, query_id="q")
+ prepare.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("pending", {"query_status": {"id": "q", "complete": False}}, InsightSeriesNotReady),
+ ("error", {"query_status": {"id": "q", "error": True}}, ValueError),
+ ("empty", {"results": []}, None),
+ ("top_level_error", {"results": None, "error": "Invalid query"}, ValueError),
+ ("missing_status", {"results": None}, ValueError),
+ ]
+ )
+ def test_pending_is_distinct_from_empty_and_error(
+ self, _name: str, response: dict[str, Any], error: type[Exception] | None
+ ) -> None:
+ with patch("posthog.api.services.query.process_query_dict", return_value=response):
+ if error:
+ with self.assertRaises(error):
+ insight_series._run(cast(Team, None), cast(User, None), {}, None, None, raise_on_pending=True)
+ else:
+ self.assertEqual(
+ insight_series._run(cast(Team, None), cast(User, None), {}, None, None, raise_on_pending=True),
+ response,
+ )
+
@parameterized.expand(
[
("cached", True, False, False, 0.0, 0),
@@ -555,7 +597,8 @@ class TestGetInsightSeries(ClickhouseTestMixin, BaseTest):
self.assertEqual((found, enqueue.call_count), (None, 1))
- def test_an_insight_the_person_cannot_view_returns_none(self) -> None:
+ @parameterized.expand([False, True])
+ def test_an_insight_the_person_cannot_view_returns_none(self, from_child_environment: bool) -> None:
self.organization.available_product_features = [
{"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}
]
@@ -565,5 +608,24 @@ class TestGetInsightSeries(ClickhouseTestMixin, BaseTest):
team_id=self.team.id, resource="insight", resource_id=str(insight.id), access_level="none"
)
reader = self._create_user("[email protected]")
+ requesting_team = (
+ Team.objects.create(
+ organization=self.organization,
+ project_id=self.team.project_id,
+ parent_team=self.team,
+ name="Other environment",
+ )
+ if from_child_environment
+ else self.team
+ )
- self.assertIsNone(get_insight_series(team=self.team, user=reader, short_id=insight.short_id))
+ self.assertIsNone(get_insight_series(team=requesting_team, user=reader, short_id=insight.short_id))
+ self.assertEqual(
+ viewable_insight_definitions(team=requesting_team, user=reader, short_ids=[insight.short_id]), {}
+ )
+ self.assertEqual(
+ viewable_insight_definitions(team=requesting_team, user=self.user, short_ids=[insight.short_id])[
+ insight.short_id
+ ].name,
+ insight.name or insight.derived_name,
+ )
diff --git a/products/today/backend/facade/api.py b/products/today/backend/facade/api.py
index 46fe985f956b..1600ab931910 100644
--- a/products/today/backend/facade/api.py
+++ b/products/today/backend/facade/api.py
@@ -52,8 +52,10 @@ def report_figure_marks(*, team: Team, user: User, report_id: str) -> list[contr
return report_pages.figure_marks(page, artefacts, _jev(team, user, figure_sources.FIGURE_MODEL))
-def get_highlights(*, team: Team, user: User, scopes: frozenset[str] | None) -> contracts.Highlights:
- return highlights.get_highlights(team=team, user=user, scopes=scopes)
+def get_highlights(
+ *, team: Team, user: User, scopes: frozenset[str] | None, refresh: bool = False
+) -> contracts.Highlights:
+ return highlights.get_highlights(team=team, user=user, scopes=scopes, refresh=refresh)
def get_pulse_pins(*, team: Team, user: User, scopes: frozenset[str] | None) -> contracts.PulsePins:
diff --git a/products/today/backend/facade/contracts.py b/products/today/backend/facade/contracts.py
index 46cac6e6a5c0..6f001fc02219 100644
--- a/products/today/backend/facade/contracts.py
+++ b/products/today/backend/facade/contracts.py
@@ -22,6 +22,7 @@ from .enums import (
HighlightUnit,
ImpactNumberKey,
KeyClauseRole,
+ PulseSlotStatus,
)
if TYPE_CHECKING:
@@ -227,6 +228,16 @@ class HighlightPin:
status: HighlightPinStatus
+@frozen
+class PulseSlot:
+ key: str
+ name: str
+ reason: HighlightMetricReason | None = None
+ status: PulseSlotStatus = PulseSlotStatus.PENDING
+ url: str | None = None
+ interval: HighlightInterval | None = None
+
+
@dataclass(frozen=True)
class Highlights:
status: HighlightsStatus
@@ -236,6 +247,7 @@ class Highlights:
pulse: list[HighlightMetric]
items: list[HighlightItem]
pins: list[HighlightPin] = field(default_factory=list)
+ pulse_slots: list[PulseSlot] | None = None
class DailyValueNotReady(Exception):
diff --git a/products/today/backend/facade/enums.py b/products/today/backend/facade/enums.py
index 98b970a16dba..038f0359c10b 100644
--- a/products/today/backend/facade/enums.py
+++ b/products/today/backend/facade/enums.py
@@ -65,6 +65,13 @@ class HighlightPinStatus(LabeledStrEnum):
UNAVAILABLE = "unavailable", "Unavailable"
+class PulseSlotStatus(LabeledStrEnum):
+ PENDING = "pending", "Loading"
+ READY = "ready", "Ready"
+ EMPTY = "empty", "No data"
+ ERROR = "error", "Could not load"
+
+
class HighlightInterval(LabeledStrEnum):
DAY = "day", "Day"
WEEK = "week", "Week"
diff --git a/products/today/backend/feature_flags.py b/products/today/backend/feature_flags.py
index 8c42891718ce..5c02ba01e0d7 100644
--- a/products/today/backend/feature_flags.py
+++ b/products/today/backend/feature_flags.py
@@ -9,6 +9,11 @@ logger = structlog.get_logger(__name__)
TODAY_RAIL_NAV_FLAG = "today-rail-nav"
TODAY_REPORT_JEV_FLAG = "today-report-jev"
+TODAY_PROGRESSIVE_PULSE_FLAG = "today-progressive-pulse"
+
+
+def progressive_pulse_enabled(user: User, team: Team) -> bool:
+ return _flag_on(TODAY_PROGRESSIVE_PULSE_FLAG, user, team)
def is_enabled_for(user: User, team: Team) -> bool:
diff --git a/products/today/backend/logic/highlight_types.py b/products/today/backend/logic/highlight_types.py
index 06acfd95c1a9..6daa57233251 100644
--- a/products/today/backend/logic/highlight_types.py
+++ b/products/today/backend/logic/highlight_types.py
@@ -27,6 +27,7 @@ from ..facade.contracts import (
HighlightContext,
HighlightFindings,
HighlightSource,
+ PulseSlot,
)
# Short names for Today's own sources in today_highlights/. Other products use the facade contract.
@@ -57,12 +58,14 @@ class Context:
today: date,
scopes: frozenset[str] | None = None,
pinned: frozenset[str] = frozenset(),
+ publish_pulse: Callable[[HighlightFindings, list[PulseSlot]], None] | None = None,
) -> None:
self.team = team
self.user = user
self.today = today
self.scopes = scopes
self.pinned = pinned
+ self.publish_pulse = publish_pulse
self.failures: list[str] = []
self.access = UserAccessControl(user=user, team=team)
diff --git a/products/today/backend/logic/highlights.py b/products/today/backend/logic/highlights.py
index 7bfb8734a2d3..321ec97d2bfe 100644
--- a/products/today/backend/logic/highlights.py
+++ b/products/today/backend/logic/highlights.py
@@ -9,6 +9,7 @@ from uuid import UUID
from django.conf import settings
from django.core.cache import cache
+from django.db import transaction
from django.utils import timezone
import structlog
@@ -17,6 +18,7 @@ from pydantic import TypeAdapter, ValidationError
from temporalio.common import WorkflowIDReusePolicy
from posthog.clickhouse.query_tagging import Feature, Product, tags_context
+from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from posthog.models import Team, User
from posthog.temporal.common.client import sync_connect
@@ -31,7 +33,9 @@ from ..facade.enums import (
HighlightPinStatus,
HighlightPriority,
HighlightsStatus,
+ PulseSlotStatus,
)
+from ..feature_flags import progressive_pulse_enabled
from ..models import TodayHighlights
from ..temporal.inputs import (
HIGHLIGHTS_WORKFLOW_NAME,
@@ -62,10 +66,24 @@ _FINDINGS = TypeAdapter(HighlightFindings)
_HIGHLIGHTS = TypeAdapter(contracts.Highlights)
-def get_highlights(*, team: Team, user: User, scopes: frozenset[str] | None) -> contracts.Highlights:
+@frozen
+class PulseSourceProgress:
+ found: HighlightFindings
+ slots: list[contracts.PulseSlot]
+
+
+_PROGRESS = TypeAdapter(dict[str, PulseSourceProgress])
+
+
+def get_highlights(
+ *, team: Team, user: User, scopes: frozenset[str] | None, refresh: bool = False
+) -> contracts.Highlights:
now = timezone.now()
day = now.astimezone(team.timezone_info).date()
key = _key(team, user, scopes)
+ progressive = progressive_pulse_enabled(user, team)
+ if progressive:
+ key = hashlib.sha256(f"{key}:progressive".encode()).hexdigest()[:32]
rows = (
TodayHighlights.objects.for_team(team.id)
.filter(user_id=user.id, key=key, local_day=day)
@@ -73,8 +91,16 @@ def get_highlights(*, team: Team, user: User, scopes: frozenset[str] | None) ->
)
latest = rows.first()
# A row with a pins key puts pinned tiles first, so compute it again with suggestions only.
- if latest is None or latest.pins_key or _is_due(latest, now):
- _start(team, user, scopes, day, key, after=latest.id if latest else None)
+ retry_failed = (
+ progressive
+ and refresh
+ and latest is not None
+ and latest.failed_sources
+ and latest.status != HighlightsStatus.COMPUTING
+ and cache.add(f"today_highlights:retry:{team.id}:{user.id}", True, FAILED_CACHE_SECONDS)
+ )
+ if latest is None or latest.pins_key or _is_due(latest, now) or retry_failed:
+ _start(team, user, scopes, day, key, after=latest.id if latest else None, progressive=progressive)
status = HighlightsStatus.COMPUTING
else:
status = HighlightsStatus(latest.status)
@@ -83,11 +109,34 @@ def get_highlights(*, team: Team, user: User, scopes: frozenset[str] | None) ->
if latest is not None and latest.status == HighlightsStatus.READY
else rows.filter(status=HighlightsStatus.READY).first()
)
+ if progressive and latest is not None and latest.content.get("pulse_slots"):
+ with suppress(ValidationError):
+ content = _HIGHLIGHTS.validate_python(latest.content)
+ if status == HighlightsStatus.COMPUTING and ready is not None and ready.id != latest.id:
+ old = _HIGHLIGHTS.validate_python(ready.content)
+ pending = {slot.key for slot in content.pulse_slots or [] if slot.status == PulseSlotStatus.PENDING}
+ retained = [metric for metric in old.pulse if metric.key in pending]
+ retained_keys = {metric.key for metric in retained}
+ content = replace(
+ content,
+ pulse=[*content.pulse, *retained],
+ pulse_slots=[
+ replace(slot, status=PulseSlotStatus.READY) if slot.key in retained_keys else slot
+ for slot in content.pulse_slots or []
+ ],
+ )
+ return replace(content, status=status)
if ready is not None:
with suppress(ValidationError):
return replace(_HIGHLIGHTS.validate_python(ready.content), status=status)
return contracts.Highlights(
- status=status, generated_at=None, day=day, refresh_at=_refresh_at(team, day), pulse=[], items=[]
+ status=status,
+ generated_at=None,
+ day=day,
+ refresh_at=_refresh_at(team, day),
+ pulse=[],
+ items=[],
+ pulse_slots=[] if progressive else None,
)
@@ -111,6 +160,7 @@ def _start(
day: date,
key: str,
after: UUID | None,
+ progressive: bool = False,
) -> None:
lock = f"today_highlights:start:{team.id}:{user.id}:{key}:{day.isoformat()}:{after}"
if not cache.add(lock, True, START_LOCK_SECONDS):
@@ -123,6 +173,7 @@ def _start(
local_day=day.isoformat(),
scopes=None if scopes is None else sorted(scopes),
source_names=[source.name for source in load_sources()],
+ progressive=progressive,
)
try:
asyncio.run(
@@ -139,6 +190,84 @@ def _start(
capture_exception(error, {"highlights_id": str(row.id), "product": "today"})
+def _pulse_content(
+ content: contracts.Highlights,
+ progress: dict[str, PulseSourceProgress],
+ previous: list[contracts.PulseSlot],
+ *,
+ complete: bool = False,
+) -> contracts.Highlights:
+ metrics = {metric.key: metric for metric in content.pulse}
+ offered = {metric.key for source in progress.values() for metric in source.found.pulse}
+ slots = {slot.key: slot for slot in previous}
+ owners: dict[str, int] = {}
+ order = {source.name: source.pulse_order for source in load_sources()}
+ for name, source in progress.items():
+ for metric in source.found.pulse:
+ owners[metric.key] = order.get(name, 0)
+ for slot in source.slots:
+ slots[slot.key] = slot
+ owners[slot.key] = order.get(name, 0)
+ for metric in content.pulse:
+ slots[metric.key] = contracts.PulseSlot(
+ key=metric.key,
+ name=metric.name,
+ url=metric.url,
+ interval=metric.interval,
+ reason=metric.reason,
+ status=PulseSlotStatus.READY,
+ )
+ pin_order = {pin.key: index for index, pin in enumerate(content.pins)}
+ selected = [slot for slot in slots.values() if slot.key not in offered or slot.key in metrics]
+ selected.sort(
+ key=lambda slot: (
+ slot.key not in pin_order,
+ pin_order.get(slot.key, 0),
+ slot.reason == HighlightMetricReason.POPULAR,
+ owners.get(slot.key, 0),
+ )
+ )
+ if complete:
+ selected = [
+ replace(slot, status=PulseSlotStatus.ERROR) if slot.status == PulseSlotStatus.PENDING else slot
+ for slot in selected
+ ]
+ return replace(content, pulse_slots=selected)
+
+
+def publish_pulse(inputs: HighlightSourceInputs, found: HighlightFindings, slots: list[contracts.PulseSlot]) -> None:
+ if inputs.highlights_id is None:
+ return
+ with transaction.atomic():
+ row = (
+ TodayHighlights.objects.for_team(inputs.team_id)
+ .select_for_update()
+ .filter(id=inputs.highlights_id, status=HighlightsStatus.COMPUTING)
+ .first()
+ )
+ if row is None:
+ return
+ progress = _PROGRESS.validate_python(row.content.get("_pulse_sources", {}))
+ previous = progress.get(inputs.source_name)
+ if previous is not None:
+ metrics = {metric.key: metric for metric in previous.found.pulse}
+ metrics.update({metric.key: metric for metric in found.pulse})
+ found = replace(found, pulse=tuple(metrics.values()))
+ slots = list({slot.key: slot for slot in [*previous.slots, *slots]}.values())
+ progress[inputs.source_name] = PulseSourceProgress(found=found, slots=slots)
+ content = build_highlights(
+ load_sources(),
+ {name: source.found for name, source in progress.items()},
+ Team.objects.get(id=inputs.team_id),
+ row.local_day,
+ )
+ previous_slots = _HIGHLIGHTS.validate_python(row.content).pulse_slots if row.content else []
+ content = _pulse_content(content, progress, previous_slots or [])
+ row.content = _HIGHLIGHTS.dump_python(replace(content, status=HighlightsStatus.COMPUTING), mode="json")
+ row.content["_pulse_sources"] = _PROGRESS.dump_python(progress, mode="json")
+ row.save(update_fields=["content"])
+
+
def run_source(inputs: HighlightSourceInputs) -> HighlightSourceResult:
source = next((source for source in load_sources() if source.name == inputs.source_name), None)
context = Context(
@@ -147,21 +276,36 @@ def run_source(inputs: HighlightSourceInputs) -> HighlightSourceResult:
date.fromisoformat(inputs.local_day),
None if inputs.scopes is None else frozenset(inputs.scopes),
frozenset(inputs.pinned),
+ publish_pulse=partial(publish_pulse, inputs) if inputs.highlights_id is not None else None,
)
if source is None or not context.allows(*source.scopes):
return HighlightSourceResult(source_name=inputs.source_name, found=None, failed=False)
with tags_context(product=Product.TODAY, feature=Feature.QUERY, team_id=context.team.id):
found = context.guard(source.name, partial(source.find, context), HighlightFindings())
+ if context.publish_pulse is not None:
+ context.publish_pulse(found, [])
return HighlightSourceResult(
source_name=source.name, found=_FINDINGS.dump_json(found).decode(), failed=bool(context.failures)
)
def finalize(inputs: FinalizeHighlightsInputs) -> None:
- row = TodayHighlights.objects.for_team(inputs.team_id).filter(id=inputs.highlights_id).first()
+ row = (
+ TodayHighlights.objects.for_team(inputs.team_id)
+ .select_for_update()
+ .filter(id=inputs.highlights_id, status=HighlightsStatus.COMPUTING)
+ .first()
+ )
if row is None:
return
found = {result.source_name: _FINDINGS.validate_json(result.found) for result in inputs.results if result.found}
+ progress = _PROGRESS.validate_python(row.content.get("_pulse_sources", {}))
+ for name, source in progress.items():
+ final = found.get(name, source.found)
+ metrics = {metric.key: metric for metric in source.found.pulse}
+ metrics.update({metric.key: metric for metric in final.pulse})
+ found[name] = replace(final, pulse=tuple(metrics.values()))
highlights = build_highlights(
load_sources(),
found,
@@ -170,6 +314,10 @@ def finalize(inputs: FinalizeHighlightsInputs) -> None:
pins=[pin.key for pin in inputs.pins],
pinned={metric.key: metric for metric in PINNED_METRICS.validate_json(inputs.pinned)},
)
+ if "_pulse_sources" in row.content:
+ highlights = _pulse_content(
+ highlights, progress, _HIGHLIGHTS.validate_python(row.content).pulse_slots or [], complete=True
+ )
failed = [result for result in inputs.results if result.failed]
usable = bool(highlights.items or highlights.pulse)
row.status = (
diff --git a/products/today/backend/logic/today_highlights/key_metrics.py b/products/today/backend/logic/today_highlights/key_metrics.py
index e3161085b077..2f78d57c9793 100644
--- a/products/today/backend/logic/today_highlights/key_metrics.py
+++ b/products/today/backend/logic/today_highlights/key_metrics.py
@@ -1,11 +1,13 @@
import json
+from dataclasses import replace
+from time import monotonic, sleep
from urllib.parse import quote
-from products.customer_analytics.backend.facade.api import get_key_metrics
+from products.customer_analytics.backend.facade.api import get_key_metric_names, get_key_metrics
from products.customer_analytics.backend.facade.contracts import KeyMetric
from ...facade import contracts
-from ...facade.enums import HighlightLabel, HighlightMetricReason, HighlightUnit
+from ...facade.enums import HighlightLabel, HighlightMetricReason, HighlightUnit, PulseSlotStatus
from ..highlight_rules import exact
from ..highlight_trends import detect
from ..highlight_types import Candidate, Context, DailyValueNotReady, Found, Source
@@ -20,12 +22,36 @@ PULSE_DAYS = 28
def find(context: Context) -> Found:
+ slots = (
+ [contracts.PulseSlot(key=key, name=name) for key, name in get_key_metric_names(team=context.team).items()]
+ if context.publish_pulse
+ else []
+ )
+ if context.publish_pulse is not None:
+ context.publish_pulse(Found(), slots)
metrics = get_key_metrics(team=context.team, user=context.user)
+ if context.publish_pulse is not None:
+ deadline = monotonic() + 20
+ while metrics is None and monotonic() < deadline:
+ sleep(min(1.0, max(0, deadline - monotonic())))
+ metrics = get_key_metrics(team=context.team, user=context.user)
+ if metrics is None:
+ context.publish_pulse(Found(), [replace(slot, status=PulseSlotStatus.ERROR) for slot in slots])
if metrics is None:
raise DailyValueNotReady("key_metrics")
shown = [metric for metric in metrics if metric.this_week or metric.last_week]
candidates = [candidate for metric in shown if (candidate := _same_weekday(context, metric)) is not None]
- return Found(items=tuple(candidates), pulse=tuple(tile(metric) for metric in shown))
+ found = Found(items=tuple(candidates), pulse=tuple(tile(metric) for metric in shown))
+ if context.publish_pulse is not None:
+ keys = {metric.key for metric in found.pulse}
+ context.publish_pulse(
+ found,
+ [
+ replace(slot, status=PulseSlotStatus.READY if slot.key in keys else PulseSlotStatus.EMPTY)
+ for slot in slots
+ ],
+ )
+ return found
def tile(metric: KeyMetric) -> contracts.HighlightMetric:
diff --git a/products/today/backend/logic/today_highlights/routine.py b/products/today/backend/logic/today_highlights/routine.py
index a82af7ebdd53..bfe4af378285 100644
--- a/products/today/backend/logic/today_highlights/routine.py
+++ b/products/today/backend/logic/today_highlights/routine.py
@@ -1,12 +1,14 @@
from collections.abc import Collection, Iterator, Sequence
+from dataclasses import replace
from datetime import date, timedelta
from functools import partial
-from time import monotonic
+from time import monotonic, sleep
from django.db.models import Count, Max
from django.utils import timezone
from posthog.dataclasses import frozen
+from posthog.exceptions_capture import capture_exception
from posthog.models import FileSystemViewLog
from posthog.models.file_system.file_system_view_log import VIEW_DAYS_KEPT
@@ -16,11 +18,16 @@ from products.dashboards.backend.facade.api import (
dashboard_insight_tile,
dashboard_insights,
)
-from products.product_analytics.backend.facade.api import get_insight_series
-from products.product_analytics.backend.facade.contracts import InsightDashboardContext
+from products.product_analytics.backend.facade.api import (
+ get_insight_series,
+ insight_query_complete,
+ insight_series_supported,
+ viewable_insight_definitions,
+)
+from products.product_analytics.backend.facade.contracts import InsightDashboardContext, InsightSeriesNotReady
from ...facade import contracts
-from ...facade.enums import HighlightInterval, HighlightMetricReason, HighlightUnit
+from ...facade.enums import HighlightInterval, HighlightMetricReason, HighlightUnit, PulseSlotStatus
from ..highlight_rules import short
from ..highlight_series import pretty, usable_series
from ..highlight_trends import complete, detect
@@ -46,11 +53,132 @@ class ViewedItem:
popular: bool = False
+@frozen
+class PulseCandidate:
+ viewed: ViewedItem
+ short_id: str
+ dashboard: DashboardInsights | None = None
+ tile: DashboardInsightTile | None = None
+
+
def find(context: Context) -> Found:
pinned = context.pinned
+ if context.publish_pulse is not None:
+ return Found(pulse=tuple(_progressive_metrics(context, pinned, PULSE_CANDIDATES - len(pinned))))
return Found(pulse=tuple(_frecent_metrics(context, pinned, PULSE_CANDIDATES - len(pinned))))
+def _dashboard_context(candidate: PulseCandidate) -> InsightDashboardContext | None:
+ if candidate.dashboard is None:
+ return None
+ return InsightDashboardContext(
+ filters=candidate.dashboard.filters,
+ tile_filters=candidate.tile.filters_overrides if candidate.tile else None,
+ variables=candidate.dashboard.variables,
+ )
+
+
+def _progressive_metrics(context: Context, excluded: Collection[str], budget: int) -> list[contracts.HighlightMetric]:
+ if budget <= 0 or context.publish_pulse is None:
+ return []
+ deadline = monotonic() + BLOCKING_SECONDS
+ candidates: list[PulseCandidate] = []
+ for item in _candidates(context, _frecent_views(context)[:FRECENCY_CANDIDATES]):
+ if item.kind == "insight":
+ candidates.append(PulseCandidate(viewed=item, short_id=item.ref))
+ elif item.ref.isdigit():
+ dashboard = dashboard_insights(
+ team=context.team, user=context.user, dashboard_id=int(item.ref), limit=DASHBOARD_TILES_TRIED
+ )
+ if dashboard is not None:
+ candidates.extend(
+ PulseCandidate(viewed=item, short_id=tile.insight_short_id, dashboard=dashboard, tile=tile)
+ for tile in dashboard.tiles
+ )
+ definitions = viewable_insight_definitions(
+ team=context.team, user=context.user, short_ids=[candidate.short_id for candidate in candidates]
+ )
+ selected: dict[str, PulseCandidate] = {}
+ slots: dict[str, contracts.PulseSlot] = {}
+ dashboards: set[int] = set()
+ for candidate in candidates:
+ key = f"insight:{candidate.short_id}"
+ definition = definitions.get(candidate.short_id)
+ if key in excluded or key in selected or definition is None:
+ continue
+ source = definition.query.get("source", definition.query)
+ if not context.guard(
+ f"eligible insight {candidate.short_id}",
+ partial(insight_series_supported, query=definition.query, dashboard=_dashboard_context(candidate)),
+ False,
+ ):
+ continue
+ dashboard = candidate.dashboard
+ if dashboard is not None and dashboard.dashboard_id in dashboards:
+ continue
+ interval = (dashboard.filters.get("interval") if dashboard else None) or source.get("interval", "day")
+ selected[key] = candidate
+ slots[key] = contracts.PulseSlot(
+ key=key,
+ name=definition.name or "Untitled insight",
+ reason=HighlightMetricReason.POPULAR if candidate.viewed.popular else HighlightMetricReason.ROUTINE,
+ url=f"/dashboard/{dashboard.dashboard_id}" if dashboard else f"/insights/{candidate.short_id}",
+ interval=HighlightInterval(interval)
+ if source.get("kind") == "TrendsQuery" and interval in HighlightInterval.values
+ else None,
+ )
+ if dashboard is not None:
+ dashboards.add(dashboard.dashboard_id)
+ if len(selected) >= budget:
+ break
+ metrics: dict[str, contracts.HighlightMetric] = {}
+ context.publish_pulse(Found(), list(slots.values()))
+ pending = dict(selected)
+ query_ids: dict[str, str] = {}
+ while pending and monotonic() < deadline:
+ for key, candidate in list(pending.items()):
+ if monotonic() >= deadline:
+ break
+ dashboard = candidate.dashboard
+ try:
+ if key in query_ids and not insight_query_complete(team=context.team, query_id=query_ids[key]):
+ continue
+ metric = insight_metric(
+ context,
+ candidate.short_id,
+ viewed=None if candidate.viewed.popular else candidate.viewed,
+ dashboard=_dashboard_context(candidate),
+ url=slots[key].url,
+ dashboard_name=dashboard.name if dashboard else None,
+ dashboard_id=dashboard.dashboard_id if dashboard else None,
+ reason=HighlightMetricReason.POPULAR if candidate.viewed.popular else None,
+ raise_on_pending=True,
+ )
+ except InsightSeriesNotReady as error:
+ if error.query_id is None:
+ raise ValueError("Pending insight has no query ID") from error
+ query_ids[key] = error.query_id
+ continue
+ except Exception as error:
+ context.failures.append(key)
+ capture_exception(error, {"product": "today", "provider": "routine"})
+ slots[key] = replace(slots[key], status=PulseSlotStatus.ERROR)
+ else:
+ slots[key] = replace(slots[key], status=PulseSlotStatus.READY if metric else PulseSlotStatus.EMPTY)
+ if metric is not None:
+ metrics[key] = metric
+ del pending[key]
+ context.publish_pulse(Found(pulse=tuple(metrics.values())), list(slots.values()))
+ if pending:
+ sleep(min(1.0, max(0, deadline - monotonic())))
+ if pending:
+ context.failures.extend(pending)
+ for key in pending:
+ slots[key] = replace(slots[key], status=PulseSlotStatus.ERROR)
+ context.publish_pulse(Found(pulse=tuple(metrics.values())), list(slots.values()))
+ return [metrics[key] for key in selected if key in metrics]
+
+
def frecency(view_days: Collection[date], today: date) -> float:
ages = [(today - day).days for day in view_days]
return sum(0.5 ** (age / FRECENCY_HALF_LIFE_DAYS) for age in ages if 0 <= age <= FRECENCY_LOOKBACK_DAYS)
@@ -220,6 +348,7 @@ def insight_metric(
reason: HighlightMetricReason | None = None,
blocking: bool = False,
wait_until: float | None = None,
+ raise_on_pending: bool = False,
) -> contracts.HighlightMetric | None:
insight = get_insight_series(
team=context.team,
@@ -228,6 +357,7 @@ def insight_metric(
dashboard=dashboard,
blocking=blocking,
wait_until=wait_until,
+ raise_on_pending=raise_on_pending,
)
if insight is None:
return None
diff --git a/products/today/backend/presentation/serializers.py b/products/today/backend/presentation/serializers.py
index cf117f4e8843..03a8337834c6 100644
--- a/products/today/backend/presentation/serializers.py
+++ b/products/today/backend/presentation/serializers.py
@@ -23,6 +23,7 @@ from ..facade.contracts import (
PullRequestLink,
PulsePin,
PulsePins,
+ PulseSlot,
RecordingTarget,
ReportKeyClauses,
ReportPage,
@@ -42,6 +43,7 @@ from ..facade.enums import (
HighlightUnit,
ImpactNumberKey,
KeyClauseRole,
+ PulseSlotStatus,
)
@@ -439,10 +441,36 @@ class HighlightPinSerializer(DataclassSerializer):
dataclass = HighlightPin
+class PulseSlotSerializer(DataclassSerializer):
+ reason = serializers.ChoiceField(
+ choices=HighlightMetricReason.choices,
+ allow_null=True,
+ required=False,
+ help_text="Why this number is selected, when known before its value arrives.",
+ )
+ key = serializers.CharField(help_text="Stable key of the selected number.")
+ name = serializers.CharField(help_text="Title available before the number is computed.")
+ status = serializers.ChoiceField(choices=PulseSlotStatus.choices, help_text="Loading state of this number.")
+ url = serializers.CharField(allow_null=True, help_text="Path to the source, relative to the project.")
+ interval = serializers.ChoiceField(
+ choices=HighlightInterval.choices, allow_null=True, help_text="Measurement interval, when known."
+ )
+
+ class Meta:
+ dataclass = PulseSlot
+
+
+class HighlightsQuerySerializer(serializers.Serializer):
+ refresh = serializers.BooleanField(
+ default=False,
+ help_text="Retry failed sources when progressive loading is enabled. An active computation is reused.",
+ )
+
+
class HighlightsSerializer(DataclassSerializer):
status = serializers.ChoiceField(
choices=HighlightsStatus.choices,
- help_text="`ready` when the content is fresh. `computing` while new highlights are being computed: the content is the last ready highlights of the day, or empty, so poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.",
+ help_text="`ready` when the content is fresh. `computing` while new highlights are being computed: the content includes available results and, when enabled, pending number details. Otherwise it is the last ready highlights of the day, or empty. Poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.",
)
generated_at = serializers.DateTimeField(
allow_null=True, help_text="When the content was computed. Null when no highlights were computed yet today."
@@ -452,6 +480,12 @@ class HighlightsSerializer(DataclassSerializer):
help_text="Midnight at the end of `day` in the project's timezone, when the next day's highlights start. Load them again then."
)
pulse = HighlightMetricSerializer(many=True, help_text="Key metrics of the project from its settings.")
+ pulse_slots = PulseSlotSerializer(
+ many=True,
+ allow_null=True,
+ required=False,
+ help_text="Selected numbers in stable display order, including unfinished numbers. Ready values are in pulse. Null for clients using complete results.",
+ )
items = HighlightItemSerializer(
many=True,
help_text="Up to 6 items, the highest scores across sources, in display order: by `priority`, then by `score`, highest first.",
diff --git a/products/today/backend/presentation/views.py b/products/today/backend/presentation/views.py
index c3c603c370b1..0b3f9f481bd0 100644
--- a/products/today/backend/presentation/views.py
+++ b/products/today/backend/presentation/views.py
@@ -28,6 +28,7 @@ from .serializers import (
ExcerptChoiceRequestSerializer,
ExcerptChoiceSerializer,
FigureMarksSerializer,
+ HighlightsQuerySerializer,
HighlightsSerializer,
KeyClausesQuerySerializer,
PulsePinsSerializer,
@@ -104,6 +105,7 @@ class TodayViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):
raise PermissionDenied()
@validated_request(
+ query_serializer=HighlightsQuerySerializer,
responses={200: OpenApiResponse(response=HighlightsSerializer)},
summary="Get today's highlights",
description="The project's key metrics and up to 6 ranked items, each labeled risk or opportunity. Queries, rules and copy templates make every item; no AI is used. The highlights are computed in the background and stay fresh for 15 minutes per person, or 1 minute when a source failed. The call never waits: while they are computed, it returns `computing` with the last ready highlights of the day, so poll every 2 seconds until the status is `ready` or `failed`. 404 when the person does not have the new navigation. A scoped key needs query:read as well, because every highlight runs a query.",
@@ -114,7 +116,12 @@ class TodayViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):
if not api.is_enabled_for(user, self.team):
raise NotFound()
scopes = get_authenticator_scopes(request.successful_authenticator)
- found = api.get_highlights(team=self.team, user=user, scopes=None if scopes is None else frozenset(scopes))
+ found = api.get_highlights(
+ team=self.team,
+ user=user,
+ scopes=None if scopes is None else frozenset(scopes),
+ refresh=request.validated_query_data["refresh"],
+ )
return Response(HighlightsSerializer(found).data)
@validated_request(
diff --git a/products/today/backend/temporal/inputs.py b/products/today/backend/temporal/inputs.py
index d2dfce46290b..280b3a1285f9 100644
--- a/products/today/backend/temporal/inputs.py
+++ b/products/today/backend/temporal/inputs.py
@@ -20,6 +20,7 @@ class ComputeHighlightsInputs:
scopes: list[str] | None
source_names: list[str]
pins: list[PinnedNumber] = dataclasses.field(default_factory=list)
+ progressive: bool = False
@dataclasses.dataclass(frozen=True)
@@ -30,6 +31,7 @@ class HighlightSourceInputs:
scopes: list[str] | None
source_name: str
pinned: list[str] = dataclasses.field(default_factory=list)
+ highlights_id: str | None = None
@dataclasses.dataclass(frozen=True)
diff --git a/products/today/backend/temporal/workflows.py b/products/today/backend/temporal/workflows.py
index 6257c82e2698..743dfdd1ea52 100644
--- a/products/today/backend/temporal/workflows.py
+++ b/products/today/backend/temporal/workflows.py
@@ -61,6 +61,7 @@ async def _run_source(inputs: ComputeHighlightsInputs, source_name: str) -> High
scopes=inputs.scopes,
source_name=source_name,
pinned=[pin.key for pin in inputs.pins],
+ highlights_id=inputs.highlights_id if inputs.progressive else None,
),
start_to_close_timeout=HIGHLIGHT_SOURCE_TIMEOUT,
retry_policy=temporalio.common.RetryPolicy(maximum_attempts=2),
diff --git a/products/today/backend/tests/test_highlight_insights.py b/products/today/backend/tests/test_highlight_insights.py
index 1460fe294bf2..de5678a4b448 100644
--- a/products/today/backend/tests/test_highlight_insights.py
+++ b/products/today/backend/tests/test_highlight_insights.py
@@ -1,4 +1,5 @@
import re
+from collections import Counter
from datetime import UTC, date, datetime, time, timedelta
from types import SimpleNamespace
from typing import Any, cast
@@ -15,7 +16,14 @@ from parameterized import parameterized
from posthog.models import FileSystemViewLog, User
from products.dashboards.backend.facade.api import DashboardInsights, DashboardInsightTile
-from products.product_analytics.backend.facade.contracts import InsightDashboardContext, InsightSeries
+from products.product_analytics.backend.facade.contracts import (
+ InsightDashboardContext,
+ InsightSeries,
+ InsightSeriesNotReady,
+ SavedInsightDefinition,
+)
+from products.today.backend.facade.contracts import HighlightFindings, PulseSlot
+from products.today.backend.facade.enums import PulseSlotStatus
from products.today.backend.logic.highlight_rules import human, short
from products.today.backend.logic.highlight_trends import BAD_METRIC, MONEY, UNCLEAR_LABEL, UNKNOWN_DIRECTION
from products.today.backend.logic.highlight_types import Context
@@ -469,3 +477,125 @@ class TestRoutineInsights(BaseTest):
("insight:once", "popular", None),
],
)
+
+
+class TestProgressiveInsights(SimpleTestCase):
+ def test_ready_numbers_are_published_while_the_first_query_is_still_running(self) -> None:
+ clock = [0.0]
+ published: list[tuple[float, HighlightFindings, list[PulseSlot]]] = []
+ ready_at = {"slow": 8.0, "fast": 1.0, "empty": 0.0, "timeout": 30.0}
+ views = [routine.ViewedItem(kind="insight", ref=ref, days=frozenset({TODAY})) for ref in ready_at]
+ definitions = {
+ ref: SavedInsightDefinition(id=i, short_id=ref, name=ref, query={"kind": "TrendsQuery"})
+ for i, ref in enumerate(ready_at)
+ }
+ queried: list[tuple[str, float]] = []
+
+ def advance(seconds: float) -> None:
+ clock[0] += seconds
+
+ def series(**kwargs: Any) -> InsightSeries | None:
+ ref = kwargs["short_id"]
+ queried.append((ref, clock[0]))
+ self.assertFalse(kwargs["blocking"])
+ self.assertTrue(kwargs["raise_on_pending"])
+ if clock[0] < ready_at[ref]:
+ raise InsightSeriesNotReady(ref)
+ return insight({"opened": days([10.0] * 56)}, short_id=ref, name=ref) if ref != "empty" else None
+
+ context = cast(
+ Context,
+ SimpleNamespace(
+ team=None,
+ user=None,
+ today=TODAY,
+ failures=[],
+ guard=lambda name, compute, fallback: compute(),
+ publish_pulse=lambda found, slots: published.append((clock[0], found, slots)),
+ ),
+ )
+ with (
+ patch.object(routine, "_frecent_views", return_value=views),
+ patch.object(routine, "_candidates", side_effect=lambda context, own: iter(own)),
+ patch.object(routine, "viewable_insight_definitions", return_value=definitions),
+ patch.object(routine, "get_insight_series", side_effect=series),
+ patch.object(
+ routine, "insight_query_complete", side_effect=lambda **kwargs: clock[0] >= ready_at[kwargs["query_id"]]
+ ),
+ patch.object(routine, "monotonic", side_effect=lambda: clock[0]),
+ patch.object(routine, "sleep", side_effect=advance),
+ ):
+ found = routine._progressive_metrics(context, frozenset(), 6)
+
+ self.assertEqual(published[0][0], 0)
+ self.assertTrue(all(slot.status == PulseSlotStatus.PENDING for slot in published[0][2]))
+ first_number = next((at, result) for at, result, _ in published if result.pulse)
+ self.assertEqual(first_number[0], 1.0)
+ self.assertEqual(first_number[1].pulse[0].key, "insight:fast")
+ self.assertEqual([metric.key for metric in found], ["insight:slow", "insight:fast"])
+ self.assertEqual(clock[0], 20.0)
+ self.assertEqual({ref for ref, at in queried if at == 0}, set(ready_at))
+ self.assertEqual(
+ [slot.status for slot in published[-1][2]],
+ [PulseSlotStatus.READY, PulseSlotStatus.READY, PulseSlotStatus.EMPTY, PulseSlotStatus.ERROR],
+ )
+ self.assertEqual(context.failures, ["insight:timeout"])
+ self.assertEqual(Counter(ref for ref, _ in queried), {"slow": 2, "fast": 2, "empty": 1, "timeout": 1})
+ self.assertTrue(
+ all([slot.key for slot in slots] == [slot.key for slot in published[0][2]] for _, _, slots in published)
+ )
+
+ def test_a_dashboard_skips_an_unsupported_tile_before_selecting_a_loading_card(self) -> None:
+ viewed = routine.ViewedItem(kind="dashboard", ref="7", days=frozenset({TODAY}))
+ dashboard = DashboardInsights(
+ dashboard_id=7,
+ name="Growth",
+ filters={},
+ variables=None,
+ tiles=(
+ DashboardInsightTile(insight_short_id="number", filters_overrides=None),
+ DashboardInsightTile(insight_short_id="chart", filters_overrides=None),
+ ),
+ )
+ definitions = {
+ ref: SavedInsightDefinition(
+ id=i,
+ short_id=ref,
+ name=ref,
+ query={
+ "kind": "InsightVizNode",
+ "source": {
+ "kind": "TrendsQuery",
+ "series": [{"kind": "EventsNode", "event": "opened"}],
+ "trendsFilter": {"display": display},
+ },
+ },
+ )
+ for i, (ref, display) in enumerate([("number", "BoldNumber"), ("chart", "ActionsLineGraph")])
+ }
+ published: list[list[PulseSlot]] = []
+ context = cast(
+ Context,
+ SimpleNamespace(
+ team=None,
+ user=None,
+ today=TODAY,
+ failures=[],
+ guard=lambda name, compute, fallback: compute(),
+ publish_pulse=lambda found, slots: published.append(slots),
+ ),
+ )
+ with (
+ patch.object(routine, "_frecent_views", return_value=[viewed]),
+ patch.object(routine, "_candidates", side_effect=lambda context, own: iter(own)),
+ patch.object(routine, "dashboard_insights", return_value=dashboard),
+ patch.object(routine, "viewable_insight_definitions", return_value=definitions),
+ patch.object(
+ routine, "get_insight_series", return_value=insight({"opened": days([10.0] * 56)}, short_id="chart")
+ ) as query,
+ ):
+ found = routine._progressive_metrics(context, frozenset(), 6)
+ self.assertEqual([slot.key for slot in published[0]], ["insight:chart"])
+ self.assertEqual([metric.key for metric in found], ["insight:chart"])
+ query.assert_called_once()
+ self.assertEqual(query.call_args.kwargs["short_id"], "chart")
diff --git a/products/today/backend/tests/test_highlight_key_metrics.py b/products/today/backend/tests/test_highlight_key_metrics.py
index ec92961b04a2..f73034947a99 100644
--- a/products/today/backend/tests/test_highlight_key_metrics.py
+++ b/products/today/backend/tests/test_highlight_key_metrics.py
@@ -62,6 +62,7 @@ class TestKeyMetrics(SimpleTestCase):
team=SimpleNamespace(id=1),
user=SimpleNamespace(id=2),
today=TODAY,
+ publish_pulse=None,
),
)
registered = next(s for s in load_sources() if s.name == "key_metrics")
@@ -89,7 +90,10 @@ class TestKeyMetrics(SimpleTestCase):
)
def test_a_pending_calculation_asks_for_a_quick_retry(self) -> None:
- context = cast(Context, SimpleNamespace(team=SimpleNamespace(id=1), user=SimpleNamespace(id=2), today=TODAY))
+ context = cast(
+ Context,
+ SimpleNamespace(team=SimpleNamespace(id=1), user=SimpleNamespace(id=2), today=TODAY, publish_pulse=None),
+ )
registered = next(s for s in load_sources() if s.name == "key_metrics")
with patch.object(key_metrics, "get_key_metrics", return_value=None), self.assertRaises(DailyValueNotReady):
diff --git a/products/today/backend/tests/test_highlights.py b/products/today/backend/tests/test_highlights.py
index 2f5f5296d20c..96b1729e1f8e 100644
--- a/products/today/backend/tests/test_highlights.py
+++ b/products/today/backend/tests/test_highlights.py
@@ -1,15 +1,21 @@
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from datetime import UTC, date, datetime, timedelta
from functools import partial
+from queue import Queue
+from time import monotonic, sleep
from types import SimpleNamespace
from typing import Any
from zoneinfo import ZoneInfo
+import pytest
import time_machine
from posthog.test.base import BaseTest
from unittest.mock import AsyncMock, MagicMock, patch
from django.core.cache import cache
+from django.db import close_old_connections, connection, connections, transaction
from django.test import SimpleTestCase
from django.utils import timezone
@@ -34,6 +40,7 @@ from products.today.backend.facade.enums import (
HighlightPriority,
HighlightsStatus,
HighlightUnit,
+ PulseSlotStatus,
)
from products.today.backend.logic import highlight_registry, highlight_types, highlights
from products.today.backend.logic.highlight_types import Context
@@ -495,3 +502,224 @@ class TestLoadSources(SimpleTestCase):
registry = highlight_registry.load_registry.__wrapped__()
self.assertEqual(dict(registry.products), {"flags": "feature_flags", "routine": "today"})
+
+
+@time_machine.travel(datetime(2026, 10, 13, 12, 0, tzinfo=UTC), tick=False)
+class TestProgressiveHighlights(BaseTest):
+ def test_partial_values_survive_source_retries_and_final_failure(self) -> None:
+ row = TodayHighlights.objects.for_team(self.team.id).create(
+ team_id=self.team.id, user_id=self.user.id, local_day=DAY, key="progress"
+ )
+ inputs = HighlightSourceInputs(
+ team_id=self.team.id,
+ user_id=self.user.id,
+ local_day=DAY.isoformat(),
+ scopes=None,
+ source_name="routine",
+ highlights_id=str(row.id),
+ )
+ slots = [contracts.PulseSlot(key="fast", name="Fast"), contracts.PulseSlot(key="slow", name="Slow")]
+ highlights.publish_pulse(inputs, HighlightFindings(), slots)
+ highlights.publish_pulse(inputs, HighlightFindings(pulse=(metric("fast"),)), slots)
+ highlights.publish_pulse(inputs, HighlightFindings(), slots)
+ row.refresh_from_db()
+ partial = HIGHLIGHTS.validate_python(row.content)
+ self.assertEqual(partial.status, HighlightsStatus.COMPUTING)
+ self.assertEqual([m.key for m in partial.pulse], ["fast"])
+ self.assertEqual(
+ [s.status for s in partial.pulse_slots or []], [PulseSlotStatus.READY, PulseSlotStatus.PENDING]
+ )
+ highlights.finalize(
+ FinalizeHighlightsInputs(
+ team_id=self.team.id,
+ highlights_id=str(row.id),
+ results=[HighlightSourceResult(source_name="routine", found=None, failed=True)],
+ )
+ )
+ row.refresh_from_db()
+ final = HIGHLIGHTS.validate_python(row.content)
+ self.assertEqual(row.status, HighlightsStatus.READY)
+ self.assertTrue(row.failed_sources)
+ self.assertNotIn("_pulse_sources", row.content)
+ self.assertEqual([m.key for m in final.pulse], ["fast"])
+ self.assertEqual([s.status for s in final.pulse_slots or []], [PulseSlotStatus.READY, PulseSlotStatus.ERROR])
+ before = row.content
+ highlights.finalize(
+ FinalizeHighlightsInputs(
+ team_id=self.team.id,
+ highlights_id=str(row.id),
+ results=[HighlightSourceResult(source_name="routine", found=None, failed=True)],
+ )
+ )
+ row.refresh_from_db()
+ self.assertEqual(row.content, before)
+ highlights.publish_pulse(inputs, HighlightFindings(pulse=(metric("slow"),)), slots)
+ row.refresh_from_db()
+ self.assertEqual(row.content, before)
+
+ @parameterized.expand([5, 6])
+ def test_progress_keeps_source_ranking_limit_and_dates_after_failure(self, routine_count: int) -> None:
+ row = TodayHighlights.objects.for_team(self.team.id).create(
+ team_id=self.team.id, user_id=self.user.id, local_day=DAY, key="ranked"
+ )
+ inputs = HighlightSourceInputs(
+ team_id=self.team.id,
+ user_id=self.user.id,
+ local_day=DAY.isoformat(),
+ scopes=None,
+ source_name="key_metrics",
+ highlights_id=str(row.id),
+ )
+ configured = [replace(metric(f"key:{i}"), reason=HighlightMetricReason.KEY_METRIC) for i in range(4)]
+ routine = [replace(metric(f"insight:{i}"), reason=HighlightMetricReason.ROUTINE) for i in range(routine_count)]
+ popular = replace(metric("popular"), reason=HighlightMetricReason.POPULAR)
+ highlights.publish_pulse(inputs, HighlightFindings(pulse=tuple(configured)), [])
+ routine_inputs = replace(inputs, source_name="routine")
+ slots = [contracts.PulseSlot(key=m.key, name=m.name, reason=m.reason) for m in [*routine, popular]]
+ highlights.publish_pulse(routine_inputs, HighlightFindings(), slots)
+ row.refresh_from_db()
+ self.assertEqual(
+ [slot.key for slot in HIGHLIGHTS.validate_python(row.content).pulse_slots or []][:routine_count],
+ [m.key for m in routine],
+ )
+ highlights.publish_pulse(routine_inputs, HighlightFindings(pulse=(*routine, popular)), slots)
+ highlights.finalize(
+ FinalizeHighlightsInputs(
+ team_id=self.team.id,
+ highlights_id=str(row.id),
+ results=[
+ HighlightSourceResult(source_name="routine", found=None, failed=True),
+ HighlightSourceResult(source_name="key_metrics", found=None, failed=True),
+ ],
+ )
+ )
+ row.refresh_from_db()
+ final = HIGHLIGHTS.validate_python(row.content)
+ expected = [m.key for m in [*routine, *configured]][:6]
+ self.assertEqual([m.key for m in final.pulse], expected)
+ self.assertEqual([slot.key for slot in final.pulse_slots or []], expected)
+ self.assertTrue(all(m.period_starts == [DAY - timedelta(days=1)] for m in final.pulse))
+
+ def test_retry_reuses_active_workflow_and_keeps_ready_numbers(self) -> None:
+ cache.clear()
+ connect = MagicMock()
+ connect.return_value.start_workflow = AsyncMock()
+ with (
+ patch.object(highlights, "progressive_pulse_enabled", return_value=True),
+ patch.object(highlights, "sync_connect", connect),
+ ):
+ highlights.get_highlights(team=self.team, user=self.user, scopes=None)
+ rows = TodayHighlights.objects.for_team(self.team.id)
+ first = rows.get()
+ inputs = HighlightSourceInputs(
+ team_id=self.team.id,
+ user_id=self.user.id,
+ local_day=DAY.isoformat(),
+ scopes=None,
+ source_name="routine",
+ highlights_id=str(first.id),
+ )
+ highlights.publish_pulse(
+ inputs,
+ HighlightFindings(pulse=(metric("fast"),)),
+ [contracts.PulseSlot(key="fast", name="Fast"), contracts.PulseSlot(key="slow", name="Slow")],
+ )
+ highlights.finalize(
+ FinalizeHighlightsInputs(
+ team_id=self.team.id,
+ highlights_id=str(first.id),
+ results=[HighlightSourceResult(source_name="routine", found=None, failed=True)],
+ )
+ )
+ rows.filter(id=first.id).update(created_at=timezone.now() - timedelta(seconds=1))
+ ready = highlights.get_highlights(team=self.team, user=self.user, scopes=None, refresh=True)
+ self.assertEqual(ready.status, HighlightsStatus.COMPUTING)
+ self.assertEqual([m.key for m in ready.pulse], ["fast"])
+ new = rows.exclude(id=first.id).get()
+ highlights.publish_pulse(
+ replace(inputs, highlights_id=str(new.id)),
+ HighlightFindings(),
+ [contracts.PulseSlot(key="fast", name="Fast"), contracts.PulseSlot(key="slow", name="Slow")],
+ )
+ pending = highlights.get_highlights(team=self.team, user=self.user, scopes=None, refresh=True)
+ self.assertEqual([m.key for m in pending.pulse], ["fast"])
+ self.assertEqual(
+ [s.status for s in pending.pulse_slots or []], [PulseSlotStatus.READY, PulseSlotStatus.PENDING]
+ )
+ self.assertEqual(connect.return_value.start_workflow.call_count, 2)
+ self.assertTrue(connect.return_value.start_workflow.call_args.args[1].progressive)
+ retry_at = timezone.now() + timedelta(seconds=highlights.FAILED_CACHE_SECONDS + 1)
+ with time_machine.travel(timezone.now() + timedelta(seconds=30), tick=False):
+ highlights.finalize(
+ FinalizeHighlightsInputs(
+ team_id=self.team.id,
+ highlights_id=str(new.id),
+ results=[HighlightSourceResult(source_name="routine", found=None, failed=True)],
+ )
+ )
+ for _ in range(3):
+ limited = highlights.get_highlights(team=self.team, user=self.user, scopes=None, refresh=True)
+ self.assertEqual(limited.status, HighlightsStatus.FAILED)
+ self.assertEqual(rows.count(), 2)
+ self.assertEqual(connect.return_value.start_workflow.call_count, 2)
+ with time_machine.travel(retry_at, tick=False):
+ retried = highlights.get_highlights(team=self.team, user=self.user, scopes=None, refresh=True)
+ self.assertEqual(retried.status, HighlightsStatus.COMPUTING)
+ self.assertEqual(rows.count(), 3)
+ self.assertEqual(connect.return_value.start_workflow.call_count, 3)
+
+
[email protected]_db(transaction=True)
+def test_finalization_keeps_a_concurrent_publication(team: Team, user: User) -> None:
+ row = TodayHighlights.objects.for_team(team.id).create(
+ team_id=team.id, user_id=user.id, local_day=DAY, key="concurrent"
+ )
+ source = HighlightSourceInputs(
+ team_id=team.id,
+ user_id=user.id,
+ local_day=DAY.isoformat(),
+ scopes=None,
+ source_name="routine",
+ highlights_id=str(row.id),
+ )
+ final = FinalizeHighlightsInputs(
+ team_id=team.id,
+ highlights_id=str(row.id),
+ results=[HighlightSourceResult(source_name="routine", found=None, failed=True)],
+ )
+ pids: Queue[int] = Queue()
+
+ def run(action: Callable[[], None]) -> None:
+ close_old_connections()
+ try:
+ with connection.cursor() as cursor:
+ cursor.execute("SELECT pg_backend_pid()")
+ pids.put(cursor.fetchone()[0])
+ action()
+ finally:
+ connections.close_all()
+
+ def wait_for_blocked_connection(pid: int) -> None:
+ deadline = monotonic() + 10
+ while monotonic() < deadline:
+ with connection.cursor() as cursor:
+ cursor.execute("SELECT cardinality(pg_blocking_pids(%s))", [pid])
+ if cursor.fetchone()[0] > 0:
+ return
+ sleep(0.01)
+ pytest.fail("The concurrent database operation did not wait for the row lock")
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ with transaction.atomic():
+ TodayHighlights.objects.for_team(team.id).select_for_update().get(id=row.id)
+ publish = pool.submit(
+ run, partial(highlights.publish_pulse, source, HighlightFindings(pulse=(metric("fast"),)), [])
+ )
+ wait_for_blocked_connection(pids.get(timeout=10))
+ finish = pool.submit(run, partial(highlights.finalize, final))
+ wait_for_blocked_connection(pids.get(timeout=10))
+ publish.result(timeout=10)
+ finish.result(timeout=10)
+ row.refresh_from_db()
+ assert row.status == HighlightsStatus.READY
+ assert [metric.key for metric in HIGHLIGHTS.validate_python(row.content).pulse] == ["fast"]
diff --git a/products/today/backend/tests/test_workflows.py b/products/today/backend/tests/test_workflows.py
index 53ca8dc3cb39..8357785cdcfc 100644
--- a/products/today/backend/tests/test_workflows.py
+++ b/products/today/backend/tests/test_workflows.py
@@ -1,9 +1,14 @@
import uuid
+import asyncio
from collections.abc import AsyncIterator
+from threading import Event
+from typing import cast
import pytest
+from unittest.mock import patch
from django.conf import settings
+from django.utils import timezone
import pytest_asyncio
from temporalio import activity
@@ -11,6 +16,14 @@ from temporalio.exceptions import ApplicationError
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
+from posthog.models import Team, User
+from posthog.sync import database_sync_to_async
+
+from products.today.backend.facade.contracts import HighlightContext, HighlightFindings, HighlightSource, PulseSlot
+from products.today.backend.logic import highlights
+from products.today.backend.logic.highlight_types import Context
+from products.today.backend.models import TodayHighlights
+from products.today.backend.temporal.activities import finalize_highlights_activity, run_highlight_source_activity
from products.today.backend.temporal.inputs import (
HIGHLIGHTS_WORKFLOW_NAME,
ComputeHighlightsInputs,
@@ -22,6 +35,7 @@ from products.today.backend.temporal.inputs import (
ResolvePinsResult,
)
from products.today.backend.temporal.workflows import ComputeTodayHighlightsWorkflow
+from products.today.backend.tests.test_highlights import HIGHLIGHTS, metric
@pytest_asyncio.fixture(scope="module")
@@ -34,6 +48,7 @@ PINS = [PinnedNumber("insight:a"), PinnedNumber("insight:b", dashboard_id=7)]
@pytest.mark.asyncio
[email protected]("progressive", [False, True])
@pytest.mark.parametrize(
"pins,pins_fail,asked,pinned",
[
@@ -43,13 +58,19 @@ PINS = [PinnedNumber("insight:a"), PinnedNumber("insight:b", dashboard_id=7)]
],
)
async def test_sources_and_pins_are_resolved_once_and_finalized_together(
- environment: WorkflowEnvironment, pins: list[PinnedNumber], pins_fail: bool, asked: list[list[str]], pinned: str
+ environment: WorkflowEnvironment,
+ progressive: bool,
+ pins: list[PinnedNumber],
+ pins_fail: bool,
+ asked: list[list[str]],
+ pinned: str,
) -> None:
finalized: list[FinalizeHighlightsInputs] = []
resolved: list[list[str]] = []
@activity.defn(name="run_highlight_source_activity")
async def run_source(inputs: HighlightSourceInputs) -> HighlightSourceResult:
+ assert inputs.highlights_id == ("h-1" if progressive else None)
if inputs.source_name == "broken":
raise ApplicationError("The source query failed.")
return HighlightSourceResult(source_name=inputs.source_name, found="{}", failed=False)
@@ -82,6 +103,7 @@ async def test_sources_and_pins_are_resolved_once_and_finalized_together(
scopes=None,
source_names=["alerts", "broken"],
pins=pins,
+ progressive=progressive,
),
id=str(uuid.uuid4()),
task_queue=settings.GENERAL_PURPOSE_TASK_QUEUE,
@@ -100,3 +122,67 @@ async def test_sources_and_pins_are_resolved_once_and_finalized_together(
)
],
)
+
+
[email protected]_db(transaction=True)
+async def test_real_activity_publishes_before_the_workflow_finishes(
+ environment: WorkflowEnvironment, team: Team, user: User
+) -> None:
+ published = Event()
+ finish = Event()
+ day = timezone.now().astimezone(team.timezone_info).date()
+
+ def create_row() -> TodayHighlights:
+ return TodayHighlights.objects.for_team(team.id).create(
+ team_id=team.id, user_id=user.id, local_day=day, key="progress-test"
+ )
+
+ row = await database_sync_to_async(create_row)()
+
+ def find(context: HighlightContext) -> HighlightFindings:
+ publisher = cast(Context, context).publish_pulse
+ assert publisher is not None
+ publisher(
+ HighlightFindings(pulse=(metric("fast"),)),
+ [PulseSlot(key="fast", name="Fast"), PulseSlot(key="slow", name="Slow")],
+ )
+ published.set()
+ assert finish.wait(15)
+ return HighlightFindings(pulse=(metric("fast"), metric("slow")))
+
+ with patch.object(highlights, "load_sources", return_value=(HighlightSource(name="routine", find=find),)):
+ async with Worker(
+ environment.client,
+ task_queue=settings.GENERAL_PURPOSE_TASK_QUEUE,
+ workflows=[ComputeTodayHighlightsWorkflow],
+ activities=[run_highlight_source_activity, finalize_highlights_activity],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ handle = await environment.client.start_workflow(
+ HIGHLIGHTS_WORKFLOW_NAME,
+ ComputeHighlightsInputs(
+ team_id=team.id,
+ user_id=user.id,
+ highlights_id=str(row.id),
+ local_day=day.isoformat(),
+ scopes=None,
+ source_names=["routine"],
+ progressive=True,
+ ),
+ id=str(uuid.uuid4()),
+ task_queue=settings.GENERAL_PURPOSE_TASK_QUEUE,
+ )
+ try:
+ assert await asyncio.to_thread(published.wait, 15)
+ await database_sync_to_async(row.refresh_from_db)()
+ content = HIGHLIGHTS.validate_python(row.content)
+ assert row.status == "computing"
+ assert [m.key for m in content.pulse] == ["fast"]
+ assert [slot.status for slot in content.pulse_slots or []] == ["ready", "pending"]
+ finally:
+ finish.set()
+ await handle.result()
+ await database_sync_to_async(row.refresh_from_db)()
+ assert row.status == "ready"
+ assert [m.key for m in HIGHLIGHTS.validate_python(row.content).pulse] == ["fast", "slow"]
diff --git a/products/today/frontend/generated/api.schemas.ts b/products/today/frontend/generated/api.schemas.ts
index 76bd8aa5798e..80e8b3211460 100644
--- a/products/today/frontend/generated/api.schemas.ts
+++ b/products/today/frontend/generated/api.schemas.ts
@@ -191,6 +191,53 @@ export interface HighlightMetricApi {
movers: HighlightMoverApi[]
}
+/**
+ * * `pending` - Loading
+ * * `ready` - Ready
+ * * `empty` - No data
+ * * `error` - Could not load
+ */
+export type PulseSlotStatusEnumApi = (typeof PulseSlotStatusEnumApi)[keyof typeof PulseSlotStatusEnumApi]
+
+export const PulseSlotStatusEnumApi = {
+ Pending: 'pending',
+ Ready: 'ready',
+ Empty: 'empty',
+ Error: 'error',
+} as const
+
+export interface PulseSlotApi {
+ /** Why this number is selected, when known before its value arrives.
+ *
+ * * `routine` - You open this often
+ * * `dashboard` - On a dashboard you open often
+ * * `key_metric` - Project key metric
+ * * `popular` - Opened often in this project */
+ reason?: HighlightMetricReasonEnumApi | null
+ /** Stable key of the selected number. */
+ key: string
+ /** Title available before the number is computed. */
+ name: string
+ /** Loading state of this number.
+ *
+ * * `pending` - Loading
+ * * `ready` - Ready
+ * * `empty` - No data
+ * * `error` - Could not load */
+ status: PulseSlotStatusEnumApi
+ /**
+ * Path to the source, relative to the project.
+ * @nullable
+ */
+ url: string | null
+ /** Measurement interval, when known.
+ *
+ * * `day` - Day
+ * * `week` - Week
+ * * `month` - Month */
+ interval: CalendarUnitEnumApi | null
+}
+
/**
* * `risk` - Risk
* * `opportunity` - Opportunity
@@ -279,7 +326,7 @@ export interface HighlightPinApi {
}
export interface HighlightsApi {
- /** `ready` when the content is fresh. `computing` while new highlights are being computed: the content is the last ready highlights of the day, or empty, so poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.
+ /** `ready` when the content is fresh. `computing` while new highlights are being computed: the content includes available results and, when enabled, pending number details. Otherwise it is the last ready highlights of the day, or empty. Poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.
*
* * `computing` - Computing
* * `ready` - Ready
@@ -296,6 +343,11 @@ export interface HighlightsApi {
refresh_at: string
/** Key metrics of the project from its settings. */
pulse: HighlightMetricApi[]
+ /**
+ * Selected numbers in stable display order, including unfinished numbers. Ready values are in pulse. Null for clients using complete results.
+ * @nullable
+ */
+ pulse_slots?: PulseSlotApi[] | null
/** Up to 6 items, the highest scores across sources, in display order: by `priority`, then by `score`, highest first. */
items: HighlightItemApi[]
/** The numbers the person keeps, in their order, with the state of each one. Pinned numbers lead `pulse`. */
@@ -601,6 +653,13 @@ export interface ReportPageApi {
last_seen: string | null
}
+export type TodayHighlightsRetrieveParams = {
+ /**
+ * Retry failed sources when progressive loading is enabled. An active computation is reused.
+ */
+ refresh?: boolean
+}
+
export type TodayReportsKeyClausesRetrieveParams = {
/**
* Whether to mark the impact sentence. Pass false when the page shows an impact number instead.
diff --git a/products/today/frontend/generated/api.ts b/products/today/frontend/generated/api.ts
index 50cf87c02150..fe08c48293ce 100644
--- a/products/today/frontend/generated/api.ts
+++ b/products/today/frontend/generated/api.ts
@@ -16,6 +16,7 @@ import type {
PulsePinsApi,
ReportKeyClausesApi,
ReportPageApi,
+ TodayHighlightsRetrieveParams,
TodayReportsKeyClausesRetrieveParams,
} from './api.schemas'
@@ -40,16 +41,32 @@ export const todayExcerptChoiceCreate = async (
})
}
-export const getTodayHighlightsRetrieveUrl = (projectId: string) => {
- return `/api/projects/${projectId}/today/highlights/`
+export const getTodayHighlightsRetrieveUrl = (projectId: string, params?: TodayHighlightsRetrieveParams) => {
+ const normalizedParams = new URLSearchParams()
+
+ Object.entries(params || {}).forEach(([key, value]) => {
+ if (value !== undefined) {
+ normalizedParams.append(key, value === null ? 'null' : String(value))
+ }
+ })
+
+ const stringifiedParams = normalizedParams.toString()
+
+ return stringifiedParams.length > 0
+ ? `/api/projects/${projectId}/today/highlights/?${stringifiedParams}`
+ : `/api/projects/${projectId}/today/highlights/`
}
/**
* The project's key metrics and up to 6 ranked items, each labeled risk or opportunity. Queries, rules and copy templates make every item; no AI is used. The highlights are computed in the background and stay fresh for 15 minutes per person, or 1 minute when a source failed. The call never waits: while they are computed, it returns `computing` with the last ready highlights of the day, so poll every 2 seconds until the status is `ready` or `failed`. 404 when the person does not have the new navigation. A scoped key needs query:read as well, because every highlight runs a query.
* @summary Get today's highlights
*/
-export const todayHighlightsRetrieve = async (projectId: string, options?: RequestInit): Promise<HighlightsApi> => {
- return apiMutator<HighlightsApi>(getTodayHighlightsRetrieveUrl(projectId), {
+export const todayHighlightsRetrieve = async (
+ projectId: string,
+ params?: TodayHighlightsRetrieveParams,
+ options?: RequestInit
+): Promise<HighlightsApi> => {
+ return apiMutator<HighlightsApi>(getTodayHighlightsRetrieveUrl(projectId, params), {
...options,
method: 'GET',
})
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index cef24c91c136..9a45d0ff8182 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -59708,8 +59708,56 @@ export namespace Schemas {
Failed: 'failed',
} as const;
+ /**
+ * * `pending` - Loading
+ * * `ready` - Ready
+ * * `empty` - No data
+ * * `error` - Could not load
+ */
+ export type PulseSlotStatusEnum = typeof PulseSlotStatusEnum[keyof typeof PulseSlotStatusEnum];
+
+
+ export const PulseSlotStatusEnum = {
+ Pending: 'pending',
+ Ready: 'ready',
+ Empty: 'empty',
+ Error: 'error',
+ } as const;
+
+ export interface PulseSlot {
+ /** Why this number is selected, when known before its value arrives.
+ *
+ * * `routine` - You open this often
+ * * `dashboard` - On a dashboard you open often
+ * * `key_metric` - Project key metric
+ * * `popular` - Opened often in this project */
+ reason?: HighlightMetricReasonEnum | null;
+ /** Stable key of the selected number. */
+ key: string;
+ /** Title available before the number is computed. */
+ name: string;
+ /** Loading state of this number.
+ *
+ * * `pending` - Loading
+ * * `ready` - Ready
+ * * `empty` - No data
+ * * `error` - Could not load */
+ status: PulseSlotStatusEnum;
+ /**
+ * Path to the source, relative to the project.
+ * @nullable
+ */
+ url: string | null;
+ /** Measurement interval, when known.
+ *
+ * * `day` - Day
+ * * `week` - Week
+ * * `month` - Month */
+ interval: CalendarUnitEnum | null;
+ }
+
export interface Highlights {
- /** `ready` when the content is fresh. `computing` while new highlights are being computed: the content is the last ready highlights of the day, or empty, so poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.
+ /** `ready` when the content is fresh. `computing` while new highlights are being computed: the content includes available results and, when enabled, pending number details. Otherwise it is the last ready highlights of the day, or empty. Poll again every 2 seconds until the status changes. `failed` when the last computation failed: the content is the last ready highlights of the day, or empty.
*
* * `computing` - Computing
* * `ready` - Ready
@@ -59726,6 +59774,11 @@ export namespace Schemas {
refresh_at: string;
/** Key metrics of the project from its settings. */
pulse: HighlightMetric[];
+ /**
+ * Selected numbers in stable display order, including unfinished numbers. Ready values are in pulse. Null for clients using complete results.
+ * @nullable
+ */
+ pulse_slots?: PulseSlot[] | null;
/** Up to 6 items, the highest scores across sources, in display order: by `priority`, then by `score`, highest first. */
items: HighlightItem[];
/** The numbers the person keeps, in their order, with the state of each one. Pinned numbers lead `pulse`. */
@@ -144915,6 +144968,13 @@ export namespace Schemas {
room: string;
};
+ export type TodayHighlightsRetrieveParams = {
+ /**
+ * Retry failed sources when progressive loading is enabled. An active computation is reused.
+ */
+ refresh?: boolean;
+ };
+
export type TodayReportsKeyClausesRetrieveParams = {
/**
* Whether to mark the impact sentence. Pass false when the page shows an impact number instead.