Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): resolve the recommended inbox arm and score sort from a new flag (#127490)

posthog[bot]2537f5687cad
diff --git a/products/signals/backend/personal_ranking/candidates.py b/products/signals/backend/personal_ranking/candidates.py
index 1c3a8f71b9d7..bd8d86ae9fce 100644
--- a/products/signals/backend/personal_ranking/candidates.py
+++ b/products/signals/backend/personal_ranking/candidates.py
@@ -30,7 +30,7 @@ from products.signals.backend.implementation_pr import (
from products.signals.backend.models import AutonomyPriority, SignalReport, SignalReportArtefact
from products.signals.backend.personal_ranking.config import CandidatePool, PersonalRankingConfig, candidate_pools
from products.signals.backend.ranking.access import can_view_inbox_ranking
-from products.signals.backend.ranking.inbox_sort import InboxRanker, latest_ranking_value, resolve_recommended_ranker
+from products.signals.backend.ranking.inbox_sort import InboxRanker, latest_ranking_value, resolve_recommended_arm
from products.signals.backend.ranking.report_predictions import ReportPredictions, read_report_predictions
from products.signals.backend.ranking.staleness import EDIT_ARTEFACT_TYPES
from products.signals.backend.report_claims import reports_claimed_for_user
@@ -53,7 +53,6 @@ _OPEN_STATUSES = (SignalReport.Status.READY, SignalReport.Status.PENDING_INPUT)
# The recommended order is computed here, so the `for_you` pool reads at most this many reports to sort.
_FOR_YOU_SCAN_LIMIT = 300
_TITLE_MAX_CHARS = 200
-_DECILES = 10
@frozen
@@ -82,7 +81,8 @@ class Candidate:
# Compares field by field in declaration order, so the order of the fields is the sort order.
@frozen(order=True)
class _RecommendedSortKey:
- decile_rank: int
+ unscored: bool
+ highest_score_first: float
priority_rank: int
status_rank: int
newest_first: float
@@ -94,13 +94,6 @@ def _priority_rank(priority: str | None) -> int:
return values.index(priority) if priority in values else len(values)
-def _decile(probability: float | None) -> int | None:
- """The same buckets as `score_decile` in the database."""
- if probability is None:
- return None
- return sum(1 for bound in range(1, _DECILES) if probability * _DECILES >= bound)
-
-
def load_predictions(team_id: int, report_ids: Sequence[str]) -> dict[str, ReportPredictions]:
"""The served predictions of each report, in two queries for the whole set."""
if not report_ids:
@@ -140,7 +133,7 @@ def _recommended_for_you(team: Team, user: User, since: datetime.datetime, size:
)
if not reports:
return []
- ranker = resolve_recommended_ranker(team, user, send_feature_flag_events=False)
+ ranker = resolve_recommended_arm(team, user, send_feature_flag_events=False).ranker
return recommended_order(team.id, ranker, reports)[:size]
@@ -152,10 +145,11 @@ def recommended_order(team_id: int, ranker: InboxRanker, reports: Sequence[Signa
def sort_key(report: SignalReport) -> _RecommendedSortKey:
report_id = str(report.id)
- decile = _decile(predictions[report_id].probability(ranker)) if ranker != "priority" else None
+ score = predictions[report_id].probability(ranker) if ranker != "priority" else None
return _RecommendedSortKey(
# Unscored reports sort after every scored report.
- decile_rank=_DECILES if decile is None else _DECILES - 1 - decile,
+ unscored=ranker != "priority" and score is None,
+ highest_score_first=-score if score is not None else 0.0,
priority_rank=_priority_rank(priorities.get(report_id)),
status_rank=0 if report.status == SignalReport.Status.READY else 1,
newest_first=-report.updated_at.timestamp(),
@@ -170,7 +164,7 @@ def _ids(queryset: QuerySet[SignalReport], size: int) -> list[str]:
def _scored_pool(queryset: QuerySet[SignalReport], head: str, size: int) -> list[str]:
- """Reports with a current score for `head`, the highest first. Unscored and stale reports drop out, so EU gives none."""
+ """Reports with a current score for `head`, the highest first. Unscored and stale reports drop out."""
return _ids(
queryset.annotate(_pool_score=latest_ranking_value(head, "scores"))
.filter(_pool_score__isnull=False)
diff --git a/products/signals/backend/ranking/inbox_sort.py b/products/signals/backend/ranking/inbox_sort.py
index 79c698ed3102..147844bbe4c9 100644
--- a/products/signals/backend/ranking/inbox_sort.py
+++ b/products/signals/backend/ranking/inbox_sort.py
@@ -1,40 +1,32 @@
-"""Resolve the `recommended` inbox sort to a ranker, and give each row a coarse merge key.
+"""Resolve the `recommended` inbox sort to an arm, and give each row a merge key.
-A multivariate flag keyed by the requesting user picks the ranker. The variants `open`, `action` and `fix` order
-by the served outcome of the same name first. `control`, an unknown variant, or a flag that is off or fails to evaluate
-gives the priority order. The EU instance gives the priority order without evaluating the flag.
+A multivariate flag keyed by the requesting user picks the arm. An arm is a ranker and a pill setting:
-The merge key lets a client merge rows from different rankers without the raw score. It is the score
-decile then the priority rank, so it never says more about the score than one decimal place. Each
-head ranker sorts by the same decile, so the row order does not say more either. The database
-computes the decile with `score_decile`, and `inbox_merge_key` reads that value.
+| Variant | Ranker | Pill |
+|----------------|----------|------|
+| `control` | priority | no |
+| `pills` | priority | yes |
+| `action_pills` | action | yes |
+
+An unknown variant, or a flag that is off or fails to evaluate, gives `control`.
+
+A head ranker sorts by the served score of its outcome, highest first, then in priority order. Unscored and stale
+reports come after every scored report, in priority order. The merge key follows the same order, so a client can merge
+rows from different state requests: `1 - score` for a scored report, `2 + priority rank` for an unscored report, and
+the priority rank alone for the priority ranker.
"""
from typing import Literal, get_args
-from django.contrib.postgres.fields import ArrayField
-from django.db.models import (
- Case,
- CharField,
- F,
- FloatField,
- Func,
- IntegerField,
- JSONField,
- OuterRef,
- Subquery,
- Value,
- When,
-)
-from django.db.models.expressions import Combinable
+from django.db.models import Case, CharField, F, FloatField, Func, JSONField, OuterRef, Subquery, Value, When
from django.db.models.functions import Cast
import structlog
import posthoganalytics
+from posthog.dataclasses import frozen
from posthog.models.team import Team
from posthog.models.user import User
-from posthog.utils import get_instance_region
from products.signals.backend.models import AutonomyPriority, SignalReportArtefact
from products.signals.backend.ranking.public_outcomes import PublicOutcomeName
@@ -42,7 +34,7 @@ from products.signals.backend.ranking.staleness import annotate_stale_score
logger = structlog.get_logger(__name__)
-RECOMMENDED_INBOX_SORT_FLAG = "signals-inbox-recommended-sort"
+RECOMMENDED_INBOX_SORT_FLAG = "signals-inbox-ranking-pills"
HeadRanker = PublicOutcomeName
InboxRanker = Literal["priority", HeadRanker]
@@ -55,18 +47,28 @@ RANKER_ORDERINGS: dict[InboxRanker, str] = {
**{head: f"-ranking_{head},{PRIORITY_ORDERING}" for head in HEAD_RANKERS},
}
-_DECILES = 10
_PRIORITY_SLOTS = len(AutonomyPriority.values) + 1
+_UNSCORED_MERGE_KEY_BASE = 2.0
+
+
+@frozen
+class RecommendedArm:
+ ranker: InboxRanker
+ ranking_pill: bool
-def resolve_recommended_ranker(team: Team, user: User, *, send_feature_flag_events: bool = True) -> InboxRanker:
+CONTROL_ARM = RecommendedArm(ranker="priority", ranking_pill=False)
+RECOMMENDED_ARMS: dict[str, RecommendedArm] = {
+ "control": CONTROL_ARM,
+ "pills": RecommendedArm(ranker="priority", ranking_pill=True),
+ "action_pills": RecommendedArm(ranker="action", ranking_pill=True),
+}
+
+
+def resolve_recommended_arm(team: Team, user: User, *, send_feature_flag_events: bool = True) -> RecommendedArm:
"""`send_feature_flag_events=False` reads the variant without an exposure, for a caller that does not show the inbox."""
if not user.distinct_id:
- return "priority"
- # EU reports have no ranking scores, so a head variant there sorts the same as `control`. Evaluating the
- # flag would still log an exposure, and those no-difference exposures would dilute the US experiment.
- if get_instance_region() == "EU":
- return "priority"
+ return CONTROL_ARM
try:
variant = posthoganalytics.get_feature_flag(
RECOMMENDED_INBOX_SORT_FLAG,
@@ -79,11 +81,8 @@ def resolve_recommended_ranker(team: Team, user: User, *, send_feature_flag_even
)
except Exception:
logger.exception("signals.inbox_sort.flag_failed", team_id=team.id, user_id=user.id)
- return "priority"
- for head in HEAD_RANKERS:
- if variant == head:
- return head
- return "priority"
+ return CONTROL_ARM
+ return RECOMMENDED_ARMS.get(variant, CONTROL_ARM) if isinstance(variant, str) else CONTROL_ARM
def latest_ranking_value(head: str, key: Literal["scores", "lifts"]) -> Subquery:
@@ -137,18 +136,15 @@ def latest_ranking_value(head: str, key: Literal["scores", "lifts"]) -> Subquery
)
-def score_decile(score: Combinable) -> Func:
- """The score decile, 0 to 9. A score below 0 gives 0, a score of 0.9 or more gives 9, and NULL stays NULL.
-
- `width_bucket` reads the score once. A CASE repeats the score in each branch, so when the score is a
- correlated subquery, Postgres runs the subquery once per branch.
- """
- thresholds = Value([float(bound) for bound in range(1, _DECILES)], output_field=ArrayField(FloatField()))
- return Func(score * _DECILES, thresholds, function="width_bucket", output_field=IntegerField())
+def _clamped_priority_rank(priority_rank: int | None) -> int:
+ return _PRIORITY_SLOTS - 1 if priority_rank is None else min(max(priority_rank, 0), _PRIORITY_SLOTS - 1)
-def inbox_merge_key(decile: int | None, priority_rank: int | None) -> int:
- """Lower keys rank first. A report without a score sorts after every scored report."""
- decile_rank = _DECILES if decile is None else _DECILES - 1 - decile
- priority = _PRIORITY_SLOTS - 1 if priority_rank is None else min(max(priority_rank, 0), _PRIORITY_SLOTS - 1)
- return decile_rank * _PRIORITY_SLOTS + priority
+def inbox_merge_key(ranker: InboxRanker, score: float | None, priority_rank: int | None) -> float:
+ """Lower keys rank first. For a head ranker, a report without a score sorts after every scored report."""
+ priority = _clamped_priority_rank(priority_rank)
+ if ranker == "priority":
+ return float(priority)
+ if score is None:
+ return _UNSCORED_MERGE_KEY_BASE + priority
+ return 1.0 - min(max(score, 0.0), 1.0)
diff --git a/products/signals/backend/serializers.py b/products/signals/backend/serializers.py
index 6d3bd282c8e2..26321da5191b 100644
--- a/products/signals/backend/serializers.py
+++ b/products/signals/backend/serializers.py
@@ -2557,10 +2557,11 @@ class SignalReportListSerializer(SignalReportSerializer):
)
inbox_merge_key = serializers.SerializerMethodField(
help_text=(
- "Coarse sort key from the ranker named by the list response's ranker field, for sort=recommended "
- "only. Lower keys rank first. It is the score decile of the ranker's outcome (open, action or fix), "
- "then the priority as a tiebreak. The priority ranker uses the priority only. "
- "Reports without a usable score sort after scored ones. Null for every other sort."
+ "Sort key from the ranker named by the list response's ranker field, for sort=recommended only. "
+ "Lower keys rank first, so rows from several requests merge in the order of one request. "
+ "For an outcome ranker (open, action or fix), a scored report gets 1 minus its score, from 0 to 1, "
+ "and a report without a usable score gets 2 plus its priority rank. The priority ranker uses the "
+ "priority rank only, from 0 (P0) to 5 (no priority). Null for every other sort."
),
)
@@ -2568,12 +2569,12 @@ class SignalReportListSerializer(SignalReportSerializer):
fields = [*SignalReportSerializer.Meta.fields, "inbox_merge_key"]
read_only_fields = fields
- def get_inbox_merge_key(self, obj: SignalReport) -> int | None:
+ def get_inbox_merge_key(self, obj: SignalReport) -> float | None:
ranker = self.context.get("inbox_ranker")
if ranker is None:
return None
- decile = None if ranker == "priority" else getattr(obj, f"ranking_{ranker}_score_decile", None)
- return inbox_merge_key(decile, getattr(obj, "priority_sort_rank", None))
+ score = None if ranker == "priority" else getattr(obj, f"ranking_{ranker}_score", None)
+ return inbox_merge_key(ranker, score, getattr(obj, "priority_sort_rank", None))
class SignalReportSectionCountsSerializer(serializers.Serializer):
diff --git a/products/signals/backend/test/test_personal_ranking.py b/products/signals/backend/test/test_personal_ranking.py
index 8b088c5c3527..9fc507b1a320 100644
--- a/products/signals/backend/test/test_personal_ranking.py
+++ b/products/signals/backend/test/test_personal_ranking.py
@@ -398,7 +398,7 @@ class TestBuildPersonalRanking(BaseTest):
),
(
"products.signals.backend.ranking.inbox_sort.posthoganalytics.get_feature_flag",
- MagicMock(return_value="action"),
+ MagicMock(return_value="action_pills"),
),
):
patcher = patch(target, value)
diff --git a/products/signals/backend/test/test_signal_report_api.py b/products/signals/backend/test/test_signal_report_api.py
index 3e7467d5be21..b5c2f65720d3 100644
--- a/products/signals/backend/test/test_signal_report_api.py
+++ b/products/signals/backend/test/test_signal_report_api.py
@@ -1772,16 +1772,17 @@ class TestSignalReportListAPI(APIBaseTest):
@parameterized.expand(
[
- ("flag_off", {"return_value": None}, None),
- ("control", {"return_value": "control"}, None),
- ("unknown_variant", {"return_value": "test"}, None),
- ("non_recommended_head", {"return_value": "discuss"}, None),
- ("flag_error", {"side_effect": RuntimeError("flags down")}, None),
- ("eu_instance", {"return_value": "fixed"}, "EU"),
+ ("flag_off", {"return_value": None}, None, False),
+ ("control", {"return_value": "control"}, None, False),
+ ("unknown_variant", {"return_value": "test"}, None, False),
+ ("old_experiment_head", {"return_value": "action"}, None, False),
+ ("flag_error", {"side_effect": RuntimeError("flags down")}, None, False),
+ ("pills", {"return_value": "pills"}, None, True),
+ ("eu_instance_pills", {"return_value": "pills"}, "EU", True),
]
)
def test_recommended_sort_uses_priority_order_unless_the_flag_serves_a_head(
- self, _name, flag_result, cloud_deployment
+ self, _name, flag_result, cloud_deployment, expected_pill
):
p2 = self._create_report(title="P2")
p0 = self._create_report(title="P0")
@@ -1798,31 +1799,31 @@ class TestSignalReportListAPI(APIBaseTest):
get_flag.configure_mock(**flag_result)
response = self.client.get(self._list_url(status="ready", sort="recommended"))
- if cloud_deployment == "EU":
- get_flag.assert_not_called()
+ get_flag.assert_called_once()
assert response.status_code == status.HTTP_200_OK
body = response.json()
assert body["ranker"] == "priority"
+ assert body["ranking_pill"] is expected_pill
assert [(r["id"], r["inbox_merge_key"]) for r in body["results"]] == [
- (str(p0.id), 60),
- (str(p2.id), 62),
- (str(no_priority.id), 65),
+ (str(p0.id), 0),
+ (str(p2.id), 2),
+ (str(no_priority.id), 5),
]
self._assert_personal_ranking_sorts_like_the_list(body)
- @parameterized.expand([("open", "open"), ("action", "action"), ("fix", "fixed")])
- def test_recommended_sort_head_variant_ranks_by_that_head_without_showing_scores_to_non_staff(self, outcome, head):
+ def test_recommended_sort_action_pills_ranks_by_the_action_score(self):
+ head = "action"
self.user.is_staff = False
self.user.save()
low_p0 = self._create_report(title="Low P0")
high_p3 = self._create_report(title="High P3")
- same_decile_p0 = self._create_report(title="Same decile P0")
+ close_score_p0 = self._create_report(title="Close score P0")
unscored_p1 = self._create_report(title="Unscored P1")
stale_p0 = self._create_report(title="Stale P0")
for report, priority in (
(low_p0, "P0"),
(high_p3, "P3"),
- (same_decile_p0, "P0"),
+ (close_score_p0, "P0"),
(unscored_p1, "P1"),
(stale_p0, "P0"),
):
@@ -1832,7 +1833,7 @@ class TestSignalReportListAPI(APIBaseTest):
for report, score in (
(low_p0, 0.1234),
(high_p3, 0.8765),
- (same_decile_p0, 0.8123),
+ (close_score_p0, 0.8123),
(stale_p0, 0.9876),
):
self._ranking_score_artefact(
@@ -1847,7 +1848,7 @@ class TestSignalReportListAPI(APIBaseTest):
with patch("products.signals.backend.ranking.inbox_sort.posthoganalytics") as analytics:
get_flag = analytics.get_feature_flag
- get_flag.return_value = outcome
+ get_flag.return_value = "action_pills"
with CaptureQueriesContext(connection) as ctx:
response = self.client.get(self._list_url(status="ready", sort="recommended"))
@@ -1861,19 +1862,19 @@ class TestSignalReportListAPI(APIBaseTest):
# The sort reads every matching report, so a score subquery repeated in the ORDER BY
# multiplies the artefact walks for the whole team.
assert list_sql[0].count("'ranking_score'") == 1
+ get_flag.assert_called_once()
assert get_flag.call_args.args[1] == self.user.distinct_id
body = response.json()
- assert body["ranker"] == outcome
+ assert body["ranker"] == "action"
+ assert body["ranking_pill"] is True
assert [(r["id"], r["inbox_merge_key"]) for r in body["results"]] == [
- (str(same_decile_p0.id), 6),
- (str(high_p3.id), 9),
- (str(low_p0.id), 48),
- (str(stale_p0.id), 60),
- (str(unscored_p1.id), 61),
+ (str(high_p3.id), pytest.approx(1 - 0.8765)),
+ (str(close_score_p0.id), pytest.approx(1 - 0.8123)),
+ (str(low_p0.id), pytest.approx(1 - 0.1234)),
+ (str(stale_p0.id), 2),
+ (str(unscored_p1.id), 3),
]
assert all(r["ranking"]["status"] == "not_available" for r in body["results"])
- for raw_score in ("0.1234", "0.8123", "0.8765", "0.9876"):
- assert raw_score not in response.content.decode()
self._assert_personal_ranking_sorts_like_the_list(body)
@parameterized.expand(
@@ -1999,11 +2000,11 @@ class TestSignalReportListAPI(APIBaseTest):
[
("priority", "priority", None, True, True, "US"),
("recommended_control", "recommended", "control", True, True, "US"),
- ("recommended_open", "recommended", "open", True, True, "US"),
- ("ai_off", "recommended", "open", False, True, "US"),
- ("autostart_off", "recommended", "open", True, False, "US"),
- ("ai_and_autostart_off", "recommended", "open", False, False, "US"),
- ("eu", "recommended", "open", True, True, "EU"),
+ ("recommended_action_pills", "recommended", "action_pills", True, True, "US"),
+ ("ai_off", "recommended", "action_pills", False, True, "US"),
+ ("autostart_off", "recommended", "action_pills", True, False, "US"),
+ ("ai_and_autostart_off", "recommended", "action_pills", False, False, "US"),
+ ("eu", "recommended", "action_pills", True, True, "EU"),
("flag_off", "recommended", None, True, True, "US"),
]
)
@@ -2029,8 +2030,8 @@ class TestSignalReportListAPI(APIBaseTest):
waiting = report_naming_me("Waits for my input", "P2", status=SignalReport.Status.PENDING_INPUT)
reviewing = report_naming_me("Names me as reviewer", "P1")
- self._ranking_score_artefact(waiting, scores={"open": 0.9})
- self._ranking_score_artefact(reviewing, scores={"open": 0.1})
+ self._ranking_score_artefact(waiting, scores={"action": 0.9})
+ self._ranking_score_artefact(reviewing, scores={"action": 0.1})
report_naming_me("Not actionable", "P0", latest_actionability="not_actionable")
report_naming_me("Already addressed", "P0", latest_already_addressed=True)
report_naming_me("Resolved", "P0", status=SignalReport.Status.RESOLVED)
@@ -2040,9 +2041,7 @@ class TestSignalReportListAPI(APIBaseTest):
self._priority_artefact(unowned, priority="P0")
first, second = (
- (waiting, reviewing)
- if sort == "recommended" and variant == "open" and region != "EU"
- else (reviewing, waiting)
+ (waiting, reviewing) if sort == "recommended" and variant == "action_pills" else (reviewing, waiting)
)
params = {"view": "for_you", "sort": sort, "created_after": (timezone.now() - timedelta(days=7)).isoformat()}
with (
diff --git a/products/signals/backend/views.py b/products/signals/backend/views.py
index 9f920f1ff0d3..501d2db09edf 100644
--- a/products/signals/backend/views.py
+++ b/products/signals/backend/views.py
@@ -166,9 +166,9 @@ from products.signals.backend.ranking.inbox_sort import (
PRIORITY_ORDERING,
RANKER_ORDERINGS,
InboxRanker,
+ RecommendedArm,
latest_ranking_value,
- resolve_recommended_ranker,
- score_decile,
+ resolve_recommended_arm,
)
from products.signals.backend.ranking.public_outcomes import MODEL_HEAD_BY_OUTCOME, PUBLIC_OUTCOME_NAMES
from products.signals.backend.ranking.served_outcomes import served_outcomes
@@ -1347,6 +1347,14 @@ class SignalReportListPagination(FacadeLimitOffsetPagination):
"inbox_merge_key belongs to this ranker."
),
}
+ paginated["properties"]["ranking_pill"] = {
+ "type": "boolean",
+ "nullable": True,
+ "description": (
+ "Whether the client shows a ranking score pill on each card, when sort=recommended. "
+ "The same flag arm sets this and the ranker. Null for every other sort."
+ ),
+ }
return paginated
@@ -1537,8 +1545,9 @@ _REPORT_LIST_PARAMETERS = [
description=(
"Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when "
"ordering is supplied. recommended resolves per user to a ranker: the priority order, or "
- "the open, action or fix outcome of the served ranking model with unscored and stale reports "
- "in priority order after the scored ones. The response's ranker field names the one used."
+ "the open, action or fix outcome of the served ranking model, highest score first, with "
+ "unscored and stale reports in priority order after the scored ones. The response's ranker "
+ "field names the one used, and its ranking_pill field says whether to show score pills."
),
),
OpenApiParameter(
@@ -1667,7 +1676,7 @@ class SignalReportViewSet(
_DEFAULT_SIGNAL_REPORT_ORDERING = "-is_suggested_reviewer,status,-updated_at"
_INBOX_SORT_ORDERINGS = {
"priority": PRIORITY_ORDERING,
- # A feature flag resolves this preset per user. See `_inbox_ranker`.
+ # A feature flag resolves this preset per user. See `_recommended_arm`.
"recommended": PRIORITY_ORDERING,
"last_updated": "-updated_at,status",
"newest": "-created_at,status,-updated_at",
@@ -2488,19 +2497,9 @@ class SignalReportViewSet(
if annotation not in ordered_fields or annotation in annotated:
continue
annotated.add(annotation)
- latest_score = latest_ranking_value(head, "scores")
- if self._inbox_ranker is not None:
- queryset = queryset.annotate(
- **{self._ranking_decile_annotation(annotation): score_decile(latest_score)}
- )
- else:
- queryset = queryset.annotate(**{annotation: latest_score})
+ queryset = queryset.annotate(**{annotation: latest_ranking_value(head, "scores")})
return queryset
- @staticmethod
- def _ranking_decile_annotation(score_annotation: str) -> str:
- return f"{score_annotation}_decile"
-
def _apply_signal_report_ranking_filters(self, queryset):
# Each named head is one correlated subquery, compared with `>=`. NULL never matches, so
# unscored and stale reports drop out of every ranking filter.
@@ -2679,17 +2678,21 @@ class SignalReportViewSet(
return self._parse_ordering_string(self._DEFAULT_SIGNAL_REPORT_ORDERING)
@cached_property
- def _inbox_ranker(self) -> InboxRanker | None:
+ def _recommended_arm(self) -> RecommendedArm | None:
params = self.request.query_params
if params.get("ordering") is not None or params.get("sort") != "recommended":
return None
- return resolve_recommended_ranker(self.team, cast(User, self.request.user))
+ return resolve_recommended_arm(self.team, cast(User, self.request.user))
+
+ @property
+ def _inbox_ranker(self) -> InboxRanker | None:
+ return self._recommended_arm.ranker if self._recommended_arm is not None else None
def _parse_signal_report_ordering(self) -> list[str]:
- ranker = self._inbox_ranker
- if ranker is not None:
- # The preset exposes only the score decile, through `inbox_merge_key`, so non-staff users may use it.
- return self._parse_ordering_string(RANKER_ORDERINGS[ranker], allow_ranking=True)
+ arm = self._recommended_arm
+ if arm is not None:
+ # The flag arm picks the ranker, so users without the model sorts may use this preset.
+ return self._parse_ordering_string(RANKER_ORDERINGS[arm.ranker], allow_ranking=True)
raw = self.request.query_params.get("ordering")
if raw is None:
inbox_sort = self.request.query_params.get("sort")
@@ -2715,11 +2718,9 @@ class SignalReportViewSet(
for clause in clauses:
name = clause.lstrip("-")
if name in ranking_annotations:
- # The recommended preset sorts by the score decile, so the order shows no more than `inbox_merge_key`.
- expression = F(self._ranking_decile_annotation(name) if self._inbox_ranker is not None else name)
# Unscored reports sort after every scored report, in both directions.
order_by.append(
- expression.desc(nulls_last=True) if clause.startswith("-") else expression.asc(nulls_last=True)
+ F(name).desc(nulls_last=True) if clause.startswith("-") else F(name).asc(nulls_last=True)
)
else:
order_by.append(clause)
@@ -2949,12 +2950,12 @@ class SignalReportViewSet(
excluded = request.validated_query_data["exclude"]
if count_only:
# A count orders nothing, so it must not record a flag exposure for the ranker.
- self._inbox_ranker = None
+ self._recommended_arm = None
with trial_state_errors():
store = trial_store_for_request(request, self.team_id)
if store is not None:
# Private trial reads compare against the priority order, never a flag arm.
- self._inbox_ranker = None
+ self._recommended_arm = None
return TrialInboxReads(self, store).list(
count_only=count_only, include_source_metadata=include_source_metadata
)
@@ -3007,6 +3008,9 @@ class SignalReportViewSet(
if page is not None:
response = self.get_paginated_response(data)
response.data["ranker"] = self._inbox_ranker
+ response.data["ranking_pill"] = (
+ self._recommended_arm.ranking_pill if self._recommended_arm is not None else None
+ )
return response
return Response(data)
diff --git a/products/signals/frontend/generated/api.schemas.ts b/products/signals/frontend/generated/api.schemas.ts
index 00bce3f162d2..b76667ac5650 100644
--- a/products/signals/frontend/generated/api.schemas.ts
+++ b/products/signals/frontend/generated/api.schemas.ts
@@ -2519,7 +2519,7 @@ export interface SignalReportListApi {
/** The plan this report leads or is a step of, with every step in it. A report that other reports link to with `part_of` is a plan. Null when the report is in no plan, and always null in list responses, because finding a plan's steps costs several queries per report. */
readonly plan: ReportPlanApi | null
/**
- * Coarse sort key from the ranker named by the list response's ranker field, for sort=recommended only. Lower keys rank first. It is the score decile of the ranker's outcome (open, action or fix), then the priority as a tiebreak. The priority ranker uses the priority only. Reports without a usable score sort after scored ones. Null for every other sort.
+ * Sort key from the ranker named by the list response's ranker field, for sort=recommended only. Lower keys rank first, so rows from several requests merge in the order of one request. For an outcome ranker (open, action or fix), a scored report gets 1 minus its score, from 0 to 1, and a report without a usable score gets 2 plus its priority rank. The priority ranker uses the priority rank only, from 0 (P0) to 5 (no priority). Null for every other sort.
* @nullable
*/
readonly inbox_merge_key: number | null
@@ -2537,6 +2537,11 @@ export interface PaginatedSignalReportListListApi {
* @nullable
*/
ranker?: string | null
+ /**
+ * Whether the client shows a ranking score pill on each card, when sort=recommended. The same flag arm sets this and the ranker. Null for every other sort.
+ * @nullable
+ */
+ ranking_pill?: boolean | null
}
/**
@@ -11196,7 +11201,7 @@ export type SignalsReportsListParams = {
*/
search?: string
/**
- * Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used.
+ * Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model, highest score first, with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used, and its ranking_pill field says whether to show score pills.
*/
sort?: string
/**
diff --git a/products/signals/frontend/inbox/types.ts b/products/signals/frontend/inbox/types.ts
index 92cff1769460..5d1cf349f04c 100644
--- a/products/signals/frontend/inbox/types.ts
+++ b/products/signals/frontend/inbox/types.ts
@@ -167,7 +167,7 @@ export interface SignalReport extends Pick<SignalReportApi, 'monitoring_started_
refund_ineligibility_reason?: string | null
/** The served ranking model's predictions. Its status says whether the report is scored, stale or unscored, or that the user cannot see the ranking model. */
ranking?: ReportRankingApi | null
- /** Coarse sort key from the list response's ranker, for the Recommended sort only. Lower keys rank first. */
+ /** Sort key from the list response's ranker, for the Recommended sort only. Lower keys rank first. */
inbox_merge_key?: number | null
}
diff --git a/products/signals/frontend/inbox/utils/reportOrdering.ts b/products/signals/frontend/inbox/utils/reportOrdering.ts
index 185c3c15749f..611493846983 100644
--- a/products/signals/frontend/inbox/utils/reportOrdering.ts
+++ b/products/signals/frontend/inbox/utils/reportOrdering.ts
@@ -69,8 +69,8 @@ export function rankingSortScore(report: SignalReport, field: InboxRankingSortFi
}
/**
- * The Recommended sort's comparator. The server orders each response by the ranker's fine-grained
- * order and gives each row only a coarse merge key, so rows with equal keys keep their input order.
+ * The Recommended sort's comparator. The server gives each row a merge key that follows the ranker's
+ * order, so rows from several responses merge in one order. Rows with equal keys keep their input order.
* A row without a key sorts last.
*/
export function compareInboxMergeKeys(a: SignalReport, b: SignalReport): number {
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 1c827e27fcdc..fd43aa60e533 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -81907,7 +81907,7 @@ export namespace Schemas {
/** The plan this report leads or is a step of, with every step in it. A report that other reports link to with `part_of` is a plan. Null when the report is in no plan, and always null in list responses, because finding a plan's steps costs several queries per report. */
readonly plan: ReportPlan | null;
/**
- * Coarse sort key from the ranker named by the list response's ranker field, for sort=recommended only. Lower keys rank first. It is the score decile of the ranker's outcome (open, action or fix), then the priority as a tiebreak. The priority ranker uses the priority only. Reports without a usable score sort after scored ones. Null for every other sort.
+ * Sort key from the ranker named by the list response's ranker field, for sort=recommended only. Lower keys rank first, so rows from several requests merge in the order of one request. For an outcome ranker (open, action or fix), a scored report gets 1 minus its score, from 0 to 1, and a report without a usable score gets 2 plus its priority rank. The priority ranker uses the priority rank only, from 0 (P0) to 5 (no priority). Null for every other sort.
* @nullable
*/
readonly inbox_merge_key: number | null;
@@ -81925,6 +81925,11 @@ export namespace Schemas {
* @nullable
*/
ranker?: string | null;
+ /**
+ * Whether the client shows a ranking score pill on each card, when sort=recommended. The same flag arm sets this and the ranker. Null for every other sort.
+ * @nullable
+ */
+ ranking_pill?: boolean | null;
}
export type SignalSourceSyncStatusEnum = typeof SignalSourceSyncStatusEnum[keyof typeof SignalSourceSyncStatusEnum];
@@ -143102,7 +143107,7 @@ export namespace Schemas {
*/
search?: string;
/**
- * Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used.
+ * Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model, highest score first, with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used, and its ranking_pill field says whether to show score pills.
*/
sort?: string;
/**
diff --git a/services/mcp/src/generated/signals/api.ts b/services/mcp/src/generated/signals/api.ts
index 50d82e028b89..03f01d0ec4b6 100644
--- a/services/mcp/src/generated/signals/api.ts
+++ b/services/mcp/src/generated/signals/api.ts
@@ -525,7 +525,7 @@ export const SignalsReportsListQueryParams = () => zod.object({
.string()
.optional()
.describe(
- "Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used."
+ "Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model, highest score first, with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used, and its ranking_pill field says whether to show score pills."
),
source_id: zod
.string()
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json
index d79314830a71..c2083aea22d4 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json
@@ -97,7 +97,7 @@
"type": "string"
},
"sort": {
- "description": "Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used.",
+ "description": "Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model, highest score first, with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used, and its ranking_pill field says whether to show score pills.",
"type": "string"
},
"source_id": {