Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): show inbox ranking predictions to agents as open, action and fix (#123005)

posthog[bot]aa89847d4d4d
diff --git a/products/signals/backend/ranking/access.py b/products/signals/backend/ranking/access.py
index 6316472734a1..44b873115e95 100644
--- a/products/signals/backend/ranking/access.py
+++ b/products/signals/backend/ranking/access.py
@@ -3,7 +3,8 @@
Staff always may. Other users need the `inbox-model-sort` flag. It is evaluated locally, with the person and group
properties the web client registers (`frontend/src/lib/utils/posthogIdentity.ts`), so the API agrees with the inbox
menu without a remote flag call on every inbox request. A flag that fails to evaluate denies access.
-Raw probabilities and model metadata stay staff only.
+Users with access see the served model key, probabilities and lifts. The manifest version and the model metadata
+stay in the `ranking_score` artefacts.
"""
import structlog
diff --git a/products/signals/backend/ranking/inbox_sort.py b/products/signals/backend/ranking/inbox_sort.py
index 19720c494c61..d80361ff35e4 100644
--- a/products/signals/backend/ranking/inbox_sort.py
+++ b/products/signals/backend/ranking/inbox_sort.py
@@ -1,7 +1,7 @@
"""Resolve the `recommended` inbox sort to a ranker, and give each row a coarse merge key.
-A multivariate flag keyed by the requesting user picks the ranker. The variants `open`, `action` and `fixed` order
-by the served head of the same name first. `control`, an unknown variant, or a flag that is off or fails to evaluate
+A multivariate flag keyed by the requesting user picks the ranker. The variants `open`, `action` and `fix` order
+by the served outcome of the same name first. `control`, an unknown variant, or a flag that is off or fails to evaluate
gives the priority order. The EU instance gives the priority order without evaluating the flag.
The merge key lets a client merge rows from different rankers without the raw score. It is the score
@@ -23,13 +23,14 @@ from posthog.models.user import User
from posthog.utils import get_instance_region
from products.signals.backend.models import AutonomyPriority
+from products.signals.backend.ranking.public_outcomes import PublicOutcomeName
logger = structlog.get_logger(__name__)
RECOMMENDED_INBOX_SORT_FLAG = "signals-inbox-recommended-sort"
-InboxRanker = Literal["priority", "open", "action", "fixed"]
-HeadRanker = Literal["open", "action", "fixed"]
+HeadRanker = PublicOutcomeName
+InboxRanker = Literal["priority", HeadRanker]
HEAD_RANKERS: tuple[HeadRanker, ...] = get_args(HeadRanker)
diff --git a/products/signals/backend/ranking/public_outcomes.py b/products/signals/backend/ranking/public_outcomes.py
new file mode 100644
index 000000000000..28c2841c52ff
--- /dev/null
+++ b/products/signals/backend/ranking/public_outcomes.py
@@ -0,0 +1,74 @@
+"""The outcomes of the ranking model that the API, the inbox sorts and the MCP tools show by name.
+
+The served model scores more heads than these. The other heads stay readable in `ranking_score` artefacts and
+through their `ranking_<head>` ordering fields, but the `ranking` field, the ranking filters and the outcomes list
+show only the outcomes here, in this order. Add an outcome here to show it everywhere at once.
+
+A public name can differ from the model head it reads. `fix` reads the `fixed` head, because `fixed` next to a
+report's status reads as "this was fixed". Training, model metadata and artefacts keep the head name.
+
+Horizon and cohort come from the served model's metadata, not from this file, so they follow the model.
+"""
+
+from typing import Literal, get_args
+
+from django.db.models import TextChoices
+
+from posthog.dataclasses import frozen
+
+PublicOutcomeName = Literal["open", "action", "fix"]
+
+
+@frozen
+class PublicOutcome:
+ outcome: PublicOutcomeName
+ model_head: str
+ label: str
+ description: str
+
+
+PUBLIC_OUTCOMES: tuple[PublicOutcome, ...] = (
+ PublicOutcome(
+ outcome="open",
+ model_head="open",
+ label="Open",
+ description="Chance that a person opens the report within the horizon. A prediction, not the report's state.",
+ ),
+ PublicOutcome(
+ outcome="action",
+ model_head="action",
+ label="Action",
+ description=(
+ "Chance that a person or their agent acts on the report within the horizon. "
+ "A prediction, not the report's state."
+ ),
+ ),
+ PublicOutcome(
+ outcome="fix",
+ model_head="fixed",
+ label="Fix",
+ description=(
+ "Chance that someone fixes the problem the report describes within the horizon. "
+ "A prediction, not the report's status."
+ ),
+ ),
+)
+
+PUBLIC_OUTCOME_NAMES: tuple[PublicOutcomeName, ...] = tuple(entry.outcome for entry in PUBLIC_OUTCOMES)
+MODEL_HEAD_BY_OUTCOME: dict[str, str] = {entry.outcome: entry.model_head for entry in PUBLIC_OUTCOMES}
+
+assert PUBLIC_OUTCOME_NAMES == get_args(PublicOutcomeName), (
+ "PUBLIC_OUTCOMES and PublicOutcomeName must list the same outcomes"
+)
+
+
+# The API's choice set for an outcome name. A Choices class gives the OpenAPI enum a stable name of its own.
+class RankingOutcome(TextChoices):
+ OPEN = "open"
+ ACTION = "action"
+ FIX = "fix"
+
+
+assert tuple(RankingOutcome.values) == PUBLIC_OUTCOME_NAMES, (
+ "RankingOutcome and PUBLIC_OUTCOMES must list the same outcomes"
+)
diff --git a/products/signals/backend/ranking/served_outcomes.py b/products/signals/backend/ranking/served_outcomes.py
new file mode 100644
index 000000000000..5b3f60484ee1
--- /dev/null
+++ b/products/signals/backend/ranking/served_outcomes.py
@@ -0,0 +1,62 @@
+"""What each public outcome of the served ranking model means for one team: horizon, cohort, base rate, readability.
+
+The values come from the model metadata that the team's latest `ranking_score` artefact copies, so the answer
+needs no model store and follows the model that scored the team's reports.
+"""
+
+from typing import Any
+
+import structlog
+
+from products.signals.backend.artefact_schemas import RankingScore
+from products.signals.backend.models import SignalReportArtefact
+from products.signals.backend.ranking.public_outcomes import PUBLIC_OUTCOMES
+
+logger = structlog.get_logger(__name__)
+
+
+def served_outcomes(team_id: int) -> tuple[str | None, list[dict[str, Any]]]:
+ """The served model key and one entry per public outcome, or `(None, [])` when the team has no readable score."""
+ from products.signals.backend.ranking.model_contract import ( # noqa: PLC0415 — keeps numpy and pandas off the API import path
+ classification_thresholds,
+ readable_head_names,
+ )
+
+ art = (
+ SignalReportArtefact.objects.filter(team_id=team_id, type=SignalReportArtefact.ArtefactType.RANKING_SCORE)
+ .order_by("-created_at")
+ .only("id", "content")
+ .first()
+ )
+ if art is None:
+ return None, []
+ try:
+ score = RankingScore.model_validate_json(art.content)
+ metadata = score.results[score.served_key].metadata
+ thresholds = classification_thresholds(metadata)
+ readable = readable_head_names(metadata)
+ except (ValueError, KeyError, TypeError, AttributeError):
+ logger.warning("signals.ranking_outcomes.invalid_content", team_id=team_id, artefact_id=str(art.id))
+ return None, []
+ recipe = metadata.get("recipe")
+ recipe_heads = recipe.get("heads") if isinstance(recipe, dict) else None
+ outcomes = []
+ for entry in PUBLIC_OUTCOMES:
+ # A model trained before the recipe was saved has no horizon or cohort, and no value may stand in for it.
+ head_recipe = recipe_heads.get(entry.model_head) if isinstance(recipe_heads, dict) else None
+ if not isinstance(head_recipe, dict):
+ head_recipe = {}
+ horizon_days = head_recipe.get("horizon_days")
+ cohort = head_recipe.get("cohort")
+ outcomes.append(
+ {
+ "outcome": entry.outcome,
+ "label": entry.label,
+ "description": entry.description,
+ "horizon_days": horizon_days if isinstance(horizon_days, int) else None,
+ "cohort": cohort if isinstance(cohort, str) else None,
+ "base_rate": thresholds.get(entry.model_head),
+ "readable": entry.model_head in readable,
+ }
+ )
+ return score.served_key, outcomes
diff --git a/products/signals/backend/serializers.py b/products/signals/backend/serializers.py
index f72c10ed9d89..550b95fcbbfb 100644
--- a/products/signals/backend/serializers.py
+++ b/products/signals/backend/serializers.py
@@ -17,6 +17,7 @@ from rest_framework.request import Request
from posthog.api.fields import RepeatedOrCommaSeparatedListField
from posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedField
+from posthog.auth import is_mcp_request
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from posthog.models import Team, User
@@ -97,6 +98,7 @@ from .models import (
)
from .pull_request_label import DEFAULT_PULL_REQUEST_LABEL
from .ranking.inbox_sort import inbox_merge_key
+from .ranking.public_outcomes import PUBLIC_OUTCOMES, RankingOutcome
from .ranking.staleness import EDIT_ARTEFACT_TYPES, is_stale_score
from .report_charts import CHART_SIZES, MAX_CHART_CAPTION_LENGTH, MAX_CHART_ID_LENGTH, MAX_CHART_TITLE_LENGTH
from .report_generation.resolve_reviewers import enrich_reviewer_dicts_with_org_members, trusted_manual_reviewer_adders
@@ -1214,44 +1216,122 @@ class ReportSourceSuggestionSerializer(serializers.Serializer):
)
-class ReportRankingSerializer(serializers.Serializer):
- served_key = serializers.CharField(
+class RankingStatus(TextChoices):
+ SCORED = "scored", "scored"
+ STALE = "stale", "stale"
+ UNSCORED = "unscored", "unscored"
+ NOT_AVAILABLE = "not_available", "not_available"
+
+
+class RankingOutcomesStatus(TextChoices):
+ SCORED = "scored", "scored"
+ UNSCORED = "unscored", "unscored"
+ NOT_AVAILABLE = "not_available", "not_available"
+
+
+class ReportRankingPredictionSerializer(serializers.Serializer):
+ outcome = serializers.ChoiceField(
+ choices=RankingOutcome.choices,
+ help_text="Outcome the model predicts. Read inbox-ranking-outcomes-list for what each outcome means.",
+ )
+ probability = serializers.FloatField(
+ help_text="The model's calibrated probability that the outcome happens within the outcome's horizon."
+ )
+ lift = serializers.FloatField(
+ allow_null=True,
+ help_text=(
+ "The probability divided by the outcome's base rate. 1.0 is an average report, 2.7 means 2.7x as "
+ "likely as the average report. Compare lifts across outcomes, not raw probabilities. Null when the "
+ "model has no saved base rate for the outcome."
+ ),
+ )
+ readable = serializers.BooleanField(
help_text=(
- "Key of the served model in the scoring pass, as `<model_name>@<model_version>`. Empty for non-staff users."
+ "True when the training run could read the outcome's holdout AUC. Treat other outcomes with caution."
)
)
- model_name = serializers.CharField(help_text="Feature family of the served model. Empty for non-staff users.")
- model_version = serializers.CharField(
- help_text="Training partition of the served model, as `YYYY-MM-DD`. Empty for non-staff users."
+
+
+class ReportRankingSerializer(serializers.Serializer):
+ status = serializers.ChoiceField(
+ choices=RankingStatus.choices,
+ help_text=(
+ "scored: the predictions describe the current text. stale: the title or summary changed after the text "
+ "the model read, so the predictions describe the old text and the model sorts and filters treat the report as "
+ "unscored. unscored: the model has not scored the report yet, or skipped it. This does not mean a low "
+ "score. not_available: you cannot see the ranking model's output, so every other field is empty."
+ ),
)
- manifest_version = serializers.CharField(
- help_text="Version of the serving manifest that chose the model. Empty for non-staff users."
+ model = serializers.CharField(
+ allow_null=True,
+ help_text=(
+ "Key of the served model that wrote the score, as `<model_name>@<model_version>`. "
+ "Do not compare predictions from different keys."
+ ),
)
- scored_at = serializers.DateTimeField(help_text="When the scoring sweep scored the report.")
- scores = serializers.DictField(
- child=serializers.FloatField(),
+ scored_at = serializers.DateTimeField(allow_null=True, help_text="When the scoring sweep scored the report.")
+ predictions = ReportRankingPredictionSerializer(
+ many=True,
help_text=(
- "Outcome head name to its calibrated probability. Empty when the served model skipped the report, "
- "and always empty for non-staff users."
+ "The served model's predictions, one per public outcome that it scored. A prediction says what may "
+ "happen, never what did: the report's status and its PR fields say whether it was resolved."
),
)
- lifts = serializers.DictField(
- child=serializers.FloatField(),
+
+
+class RankingOutcomeInfoSerializer(serializers.Serializer):
+ outcome = serializers.ChoiceField(
+ choices=RankingOutcome.choices, help_text="Name of the outcome in the `ranking` field's predictions."
+ )
+ label = serializers.CharField( # type: ignore[assignment] # field name intentionally shadows Field.label
+ help_text="Short label the inbox shows for the outcome."
+ )
+ description = serializers.CharField(help_text="What the outcome predicts, in one sentence.")
+ horizon_days = serializers.IntegerField(
+ allow_null=True,
+ help_text="Days after scoring within which the outcome counts. Null when the served model did not save it.",
+ )
+ cohort = serializers.CharField(
+ allow_null=True,
+ help_text="Name of the report set the outcome was trained on. Null when the served model did not save it.",
+ )
+ base_rate = serializers.FloatField(
+ allow_null=True,
help_text=(
- "Outcome head name to its probability divided by the head's training base rate, e.g. 2.7 means "
- "2.7x as likely as the average report. A head without a saved base rate has no entry."
+ "The outcome's base rate: a report's lift is its probability divided by this value. "
+ "Null when the served model has no saved base rate for the outcome."
),
)
- readable_heads = serializers.ListField(
- child=serializers.CharField(),
- help_text="Heads whose holdout AUC the training run could read. Treat scores of other heads with caution.",
+ readable = serializers.BooleanField(
+ help_text=(
+ "True when the training run could read the outcome's holdout AUC. Treat other outcomes with caution."
+ )
)
- stale = serializers.BooleanField(
+
+
+class RankingOutcomesResponseSerializer(serializers.Serializer):
+ status = serializers.ChoiceField(
+ choices=RankingOutcomesStatus.choices,
help_text=(
- "True when the report's title or summary was edited after the text this score read. The score "
- "describes the old text: the inbox hides its lift and the model sort treats the report as unscored."
+ "scored: the served model has scored reports in this project. unscored: no readable score yet. "
+ "not_available: you cannot see the ranking model's output."
),
)
+ model = serializers.CharField(
+ allow_null=True,
+ help_text="Key of the served model, as `<model_name>@<model_version>`, from the project's latest score.",
+ )
+ outcomes = RankingOutcomeInfoSerializer(many=True, help_text="One entry per public outcome, in the inbox order.")
+
+
+def _ranking_payload(
+ status: RankingStatus,
+ *,
+ model: str | None = None,
+ scored_at: datetime | None = None,
+ predictions: list[dict] | None = None,
+) -> dict:
+ return {"status": status.value, "model": model, "scored_at": scored_at, "predictions": predictions or []}
class SignalReportPriorityUpdateSerializer(serializers.Serializer):
@@ -1537,8 +1617,8 @@ class SignalReportSerializer(serializers.ModelSerializer):
)
ranking = serializers.SerializerMethodField(
help_text=(
- "The served model's score from the latest ranking score artefact. Null for users who are neither "
- "staff nor in the inbox model sort rollout, and null when the report has no score."
+ "The served ranking model's predictions from the latest ranking score artefact. Its status says "
+ "whether the report is scored, stale or unscored, or that the user cannot see the ranking model."
),
)
source_suggestion = serializers.SerializerMethodField(
@@ -1814,14 +1894,12 @@ class SignalReportSerializer(serializers.ModelSerializer):
value = data.get("repository")
return value if isinstance(value, str) and value else None
- @extend_schema_field(ReportRankingSerializer(allow_null=True))
- def get_ranking(self, obj: SignalReport) -> dict | None:
+ @extend_schema_field(ReportRankingSerializer)
+ def get_ranking(self, obj: SignalReport) -> dict:
request = self.context.get("request")
- if request is None:
- return None
can_view = self.context.get("can_view_inbox_ranking")
- if not (can_view() if can_view is not None else request.user.is_staff):
- return None
+ if request is None or not (can_view() if can_view is not None else request.user.is_staff):
+ return _ranking_payload(RankingStatus.NOT_AVAILABLE)
from products.signals.backend.ranking.model_contract import ( # noqa: PLC0415 — keeps numpy and pandas off the API import path
head_lifts,
@@ -1838,7 +1916,7 @@ class SignalReportSerializer(serializers.ModelSerializer):
.first()
)
if art is None:
- return None
+ return _ranking_payload(RankingStatus.UNSCORED)
prefetched_edits = getattr(obj, "prefetched_latest_edit_artefacts", None)
if prefetched_edits is not None:
latest_edit_at = prefetched_edits[0].created_at if prefetched_edits else None
@@ -1852,27 +1930,43 @@ class SignalReportSerializer(serializers.ModelSerializer):
try:
score = RankingScore.model_validate_json(art.content)
served = score.results[score.served_key]
- heads = readable_head_names(served.metadata)
- if not all(isinstance(head, str) and head for head in heads):
- raise ValueError("readable head names must be non-empty strings")
- readable_heads = sorted(heads)
+ except (ValueError, KeyError):
+ logger.warning("signals.ranking_score.invalid_content", report_id=str(obj.id), artefact_id=str(art.id))
+ return _ranking_payload(RankingStatus.UNSCORED)
+ # The ranking filters read the score alone, so bad readability or threshold metadata must not hide it here.
+ try:
+ readable_heads = readable_head_names(served.metadata)
+ except (KeyError, TypeError, AttributeError):
+ readable_heads = frozenset()
+ try:
# A score written before lifts were stored carries the metadata to compute them.
lifts = served.lifts or head_lifts(served.scores, served.metadata)
except (ValueError, KeyError, TypeError, AttributeError):
- logger.warning("signals.ranking_score.invalid_content", report_id=str(obj.id), artefact_id=str(art.id))
- return None
- is_staff = request.user.is_staff
- return {
- "served_key": score.served_key if is_staff else "",
- "model_name": served.model_name if is_staff else "",
- "model_version": served.model_version if is_staff else "",
- "manifest_version": score.manifest_version if is_staff else "",
- "scored_at": score.scored_at,
- "scores": served.scores if is_staff else {},
- "lifts": lifts,
- "readable_heads": readable_heads,
- "stale": is_stale_score(score, latest_edit_at),
- }
+ lifts = {}
+ # Agents get rounded values because extra digits suggest precision the model does not have.
+ for_agent = is_mcp_request(request)
+ predictions = []
+ for entry in PUBLIC_OUTCOMES:
+ probability = served.scores.get(entry.model_head)
+ if probability is None:
+ continue
+ lift = lifts.get(entry.model_head)
+ predictions.append(
+ {
+ "outcome": entry.outcome,
+ "probability": round(probability, 2) if for_agent else probability,
+ "lift": round(lift, 1) if for_agent and lift is not None else lift,
+ "readable": entry.model_head in readable_heads,
+ }
+ )
+ if not predictions:
+ return _ranking_payload(RankingStatus.UNSCORED)
+ return _ranking_payload(
+ RankingStatus.STALE if is_stale_score(score, latest_edit_at) else RankingStatus.SCORED,
+ model=score.served_key,
+ scored_at=score.scored_at,
+ predictions=predictions,
+ )
def get_source_products(self, obj: SignalReport) -> list[str]:
source_products_map: dict[str, list[str]] | None = self.context.get("source_products_map")
@@ -2046,7 +2140,7 @@ class SignalReportListSerializer(SignalReportSerializer):
inbox_merge_key = serializers.SerializerMethodField(
help_text=(
"Coarse sort key from the ranker named by the list response's ranker field, for sort=recommended "
- "only. Lower keys rank first. It is the score decile of the ranker's head (open, action or fixed), "
+ "only. Lower keys rank first. It is the score decile of the ranker's outcome (open, action or fix), "
"then the priority as a tiebreak. The priority ranker uses the priority only. "
"Reports without a usable score sort after scored ones. Null for every other sort."
),
diff --git a/products/signals/backend/test/test_signal_report_api.py b/products/signals/backend/test/test_signal_report_api.py
index 9d4b2b532f28..fb8e70f3f67c 100644
--- a/products/signals/backend/test/test_signal_report_api.py
+++ b/products/signals/backend/test/test_signal_report_api.py
@@ -2,7 +2,7 @@ import json
import uuid
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urlencode
import pytest
@@ -875,6 +875,7 @@ class TestSignalReportListAPI(APIBaseTest):
heads: list[dict] | None = None,
lifts: dict[str, float] | None = None,
embedding_inserted_at: datetime | None = None,
+ extra_metadata: dict | None = None,
) -> SignalReportArtefact:
served = RankingModelResult(
model_name="report_embeddings",
@@ -885,7 +886,10 @@ class TestSignalReportListAPI(APIBaseTest):
status="scored",
scores=scores or {"open": 0.5, "merged": 0.2},
lifts=lifts or {},
- metadata={"heads": heads or [{"head": "open", "readable": True}, {"head": "merged", "readable": False}]},
+ metadata={
+ "heads": heads or [{"head": "open", "readable": True}, {"head": "merged", "readable": False}],
+ **(extra_metadata or {}),
+ },
)
challenger = RankingModelResult(
model_name="report_tabular",
@@ -908,66 +912,90 @@ class TestSignalReportListAPI(APIBaseTest):
team=self.team, report=report, type=SignalReportArtefact.ArtefactType.RANKING_SCORE, content=content
)
+ _RANKING_NOT_AVAILABLE: dict[str, Any] = {
+ "status": "not_available",
+ "model": None,
+ "scored_at": None,
+ "predictions": [],
+ }
+ _RANKING_UNSCORED: dict[str, Any] = {"status": "unscored", "model": None, "scored_at": None, "predictions": []}
+
@parameterized.expand(
[
(
- "staff_sees_the_served_result",
+ "staff_sees_the_public_outcomes_of_the_served_result",
True,
None,
None,
+ None,
{"open": 2.5},
{
- "served_key": "report_embeddings@2026-09-01",
- "model_name": "report_embeddings",
- "model_version": "2026-09-01",
- "manifest_version": "manifest",
+ "status": "scored",
+ "model": "report_embeddings@2026-09-01",
"scored_at": "2026-09-20T12:00:00Z",
- "scores": {"open": 0.5, "merged": 0.2},
- "lifts": {"open": 2.5},
- "readable_heads": ["open"],
- "stale": False,
+ "predictions": [{"outcome": "open", "probability": 0.5, "lift": 2.5, "readable": True}],
},
),
(
"legacy_score_without_lifts_computes_them_from_the_metadata",
True,
None,
+ None,
[
{"head": "open", "readable": True, "refit_classification_threshold": 0.25},
{"head": "merged", "readable": False, "refit_classification_threshold": 0.0},
],
None,
{
- "served_key": "report_embeddings@2026-09-01",
- "model_name": "report_embeddings",
- "model_version": "2026-09-01",
- "manifest_version": "manifest",
+ "status": "scored",
+ "model": "report_embeddings@2026-09-01",
"scored_at": "2026-09-20T12:00:00Z",
- "scores": {"open": 0.5, "merged": 0.2},
- "lifts": {"open": 2.0},
- "readable_heads": ["open"],
- "stale": False,
+ "predictions": [{"outcome": "open", "probability": 0.5, "lift": 2.0, "readable": True}],
},
),
- ("non_staff_sees_nothing", False, None, None, None, None),
- ("invalid_content_reads_as_none", True, '{"served_key": "missing"}', None, None, None),
- ("null_readable_head_reads_as_none", True, None, [{"head": None, "readable": True}], None, None),
+ ("non_staff_without_the_flag_is_not_available", False, None, None, None, None, _RANKING_NOT_AVAILABLE),
+ (
+ "invalid_content_reads_as_unscored",
+ True,
+ '{"served_key": "missing"}',
+ None,
+ None,
+ None,
+ _RANKING_UNSCORED,
+ ),
+ (
+ "null_readable_head_keeps_the_score_with_no_readable_head",
+ True,
+ None,
+ None,
+ [{"head": None, "readable": True}],
+ None,
+ {
+ "status": "scored",
+ "model": "report_embeddings@2026-09-01",
+ "scored_at": "2026-09-20T12:00:00Z",
+ "predictions": [{"outcome": "open", "probability": 0.5, "lift": None, "readable": False}],
+ },
+ ),
+ ("only_internal_heads_read_as_unscored", True, None, {"merged": 0.2}, None, None, _RANKING_UNSCORED),
]
)
- def test_ranking_field_in_the_list_and_the_detail(self, _name, is_staff, content, heads, lifts, expected):
+ def test_ranking_field_in_the_list_and_the_detail(self, _name, is_staff, content, scores, heads, lifts, expected):
self.user.is_staff = is_staff
self.user.save()
report = self._create_report()
stale = self._ranking_score_artefact(report, scores={"open": 0.9})
SignalReportArtefact.objects.filter(pk=stale.pk).update(created_at=timezone.now() - timedelta(days=1))
- self._ranking_score_artefact(report, content=content, heads=heads, lifts=lifts)
+ self._ranking_score_artefact(report, content=content, scores=scores, heads=heads, lifts=lifts)
unscored = self._create_report(title="Unscored")
list_response = self.client.get(self._list_url())
assert list_response.status_code == status.HTTP_200_OK
rows = {row["id"]: row for row in list_response.json()["results"]}
assert rows[str(report.id)]["ranking"] == expected
- assert rows[str(unscored.id)]["ranking"] is None
+ assert rows[str(unscored.id)]["ranking"] == (
+ self._RANKING_UNSCORED if is_staff else self._RANKING_NOT_AVAILABLE
+ )
response = self.client.get(f"/api/projects/{self.team.id}/signals/reports/{report.id}/")
assert response.status_code == status.HTTP_200_OK
@@ -980,7 +1008,7 @@ class TestSignalReportListAPI(APIBaseTest):
("flag_error_fails_closed", RuntimeError("flags unavailable"), status.HTTP_400_BAD_REQUEST),
]
)
- def test_model_sort_flag_opens_lifts_but_not_raw_scores_to_non_staff(self, _name, flag, expected_status):
+ def test_model_sort_flag_opens_the_ranking_to_non_staff(self, _name, flag, expected_status):
self.user.is_staff = False
self.user.save()
high = self._create_report(title="High")
@@ -996,24 +1024,154 @@ class TestSignalReportListAPI(APIBaseTest):
assert response.status_code == expected_status
if expected_status != status.HTTP_200_OK:
assert response.json()["attr"] == "ordering"
- assert detail.json()["ranking"] is None
+ assert detail.json()["ranking"] == self._RANKING_NOT_AVAILABLE
return
rows = response.json()["results"]
assert [row["id"] for row in rows] == [str(high.id), str(low.id)]
assert detail.json()["ranking"] == rows[0]["ranking"]
assert rows[0]["ranking"] == {
- "served_key": "",
- "model_name": "",
- "model_version": "",
- "manifest_version": "",
+ "status": "scored",
+ "model": "report_embeddings@2026-09-01",
"scored_at": "2026-09-20T12:00:00Z",
- "scores": {},
- "lifts": {"open": 3.2},
- "readable_heads": ["open"],
- "stale": False,
+ "predictions": [{"outcome": "open", "probability": 0.8123, "lift": 3.2, "readable": True}],
}
- for raw_score in ("0.1234", "0.8123", "report_embeddings"):
- assert raw_score not in response.content.decode()
+ assert "manifest" not in json.dumps(rows[0]["ranking"])
+
+ @parameterized.expand([("web", False, 0.8123, 3.2678), ("mcp", True, 0.81, 3.3)])
+ def test_ranking_values_round_for_agents_only(self, _name, via_mcp, probability, lift):
+ self.user.is_staff = True
+ self.user.save()
+ report = self._create_report()
+ self._ranking_score_artefact(report, scores={"fixed": 0.8123}, lifts={"fixed": 3.2678})
+
+ with patch("products.signals.backend.serializers.is_mcp_request", return_value=via_mcp):
+ response = self.client.get(f"/api/projects/{self.team.id}/signals/reports/{report.id}/")
+
+ assert response.status_code == status.HTTP_200_OK
+ assert response.json()["ranking"]["predictions"] == [
+ {"outcome": "fix", "probability": probability, "lift": lift, "readable": False}
+ ]
+
+ @parameterized.expand(
+ [
+ ("min_lift", {"ranking_min_lift": "fix:2"}, ["high"]),
+ ("min_lift_on_two_heads", {"ranking_min_lift": "fix:1.5,open:1.5"}, ["mid"]),
+ ("min_probability", {"ranking_min_probability": "fix:0.3"}, ["high", "mid"]),
+ ("repeated_head_lift_keeps_the_strictest", {"ranking_min_lift": "fix:2,fix:1"}, ["high"]),
+ (
+ "repeated_head_probability_keeps_the_strictest",
+ {"ranking_min_probability": "fix:0.5,fix:0.2"},
+ ["high"],
+ ),
+ ]
+ )
+ def test_ranking_filters_keep_scored_reports_at_or_above_every_minimum(self, _name, query, expected):
+ self.user.is_staff = True
+ self.user.save()
+ reports = {name: self._create_report(title=name) for name in ("high", "mid", "low", "stale", "unscored")}
+ self._ranking_score_artefact(
+ reports["high"], scores={"fixed": 0.6, "open": 0.5}, lifts={"fixed": 3.0, "open": 1.0}
+ )
+ self._ranking_score_artefact(
+ reports["mid"], scores={"fixed": 0.3, "open": 0.9}, lifts={"fixed": 1.5, "open": 2.0}
+ )
+ self._ranking_score_artefact(
+ reports["low"], scores={"fixed": 0.1, "open": 0.1}, lifts={"fixed": 0.5, "open": 0.5}
+ )
+ self._ranking_score_artefact(
+ reports["stale"], scores={"fixed": 0.9, "open": 0.9}, lifts={"fixed": 4.0, "open": 4.0}
+ )
+ SignalReportArtefact.objects.create(
+ team=self.team,
+ report=reports["stale"],
+ type=SignalReportArtefact.ArtefactType.TITLE_CHANGE,
+ content=json.dumps({"old_title": "Old", "new_title": "stale"}),
+ )
+
+ response = self.client.get(self._list_url(status="ready", **query))
+
+ assert response.status_code == status.HTTP_200_OK
+ assert sorted(row["title"] for row in response.json()["results"]) == sorted(expected)
+
+ def test_ranking_filter_match_shows_as_scored_despite_bad_readable_metadata(self):
+ self.user.is_staff = True
+ self.user.save()
+ report = self._create_report()
+ self._ranking_score_artefact(report, scores={"fixed": 0.6}, heads=[{"head": None, "readable": True}])
+
+ response = self.client.get(self._list_url(status="ready", ranking_min_probability="fix:0.5"))
+
+ assert response.status_code == status.HTTP_200_OK
+ rows = response.json()["results"]
+ assert [row["id"] for row in rows] == [str(report.id)]
+ assert rows[0]["ranking"]["status"] == "scored"
+
+ @parameterized.expand(
+ [
+ ("internal_head", True, {"ranking_min_lift": "pr_merged:2"}, "ranking_min_lift"),
+ ("malformed_value", True, {"ranking_min_probability": "fix:high"}, "ranking_min_probability"),
+ ("missing_value", True, {"ranking_min_lift": "fix"}, "ranking_min_lift"),
+ ("model_head_name", True, {"ranking_min_lift": "fixed:2"}, "ranking_min_lift"),
+ ("no_access", False, {"ranking_min_lift": "fix:2"}, "ranking_min_lift"),
+ ]
+ )
+ def test_ranking_filters_reject_bad_input_and_users_without_access(self, _name, is_staff, query, attr):
+ self.user.is_staff = is_staff
+ self.user.save()
+ with patch("products.signals.backend.ranking.access.feature_enabled_or_false", return_value=False):
+ response = self.client.get(self._list_url(**query))
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert response.json()["attr"] == attr
+
+ @parameterized.expand(
+ [
+ (
+ "with_recipe",
+ {"recipe": {"heads": {"fixed": {"horizon_days": 30, "cohort": "actioned"}}}},
+ 30,
+ "actioned",
+ ),
+ ("without_recipe", {}, None, None),
+ ]
+ )
+ def test_ranking_outcomes_reads_the_served_model_of_the_latest_score(self, _name, extra_metadata, horizon, cohort):
+ self.user.is_staff = True
+ self.user.save()
+ report = self._create_report()
+ self._ranking_score_artefact(
+ report,
+ scores={"fixed": 0.4},
+ heads=[{"head": "fixed", "readable": True, "refit_classification_threshold": 0.2}],
+ extra_metadata=extra_metadata,
+ )
+
+ response = self.client.get(f"/api/projects/{self.team.id}/signals/reports/ranking_outcomes/")
+
+ assert response.status_code == status.HTTP_200_OK
+ body = response.json()
+ assert body["status"] == "scored"
+ assert body["model"] == "report_embeddings@2026-09-01"
+ assert [entry["outcome"] for entry in body["outcomes"]] == ["open", "action", "fix"]
+ assert body["outcomes"][2] | {"description": None} == {
+ "outcome": "fix",
+ "label": "Fix",
+ "description": None,
+ "horizon_days": horizon,
+ "cohort": cohort,
+ "base_rate": 0.2,
+ "readable": True,
+ }
+ assert body["outcomes"][0]["base_rate"] is None
+ assert body["outcomes"][0]["readable"] is False
+
+ @parameterized.expand([("no_access", False, "not_available"), ("no_score_yet", True, "unscored")])
+ def test_ranking_outcomes_without_a_readable_model(self, _name, is_staff, expected_status):
+ self.user.is_staff = is_staff
+ self.user.save()
+ with patch("products.signals.backend.ranking.access.feature_enabled_or_false", return_value=False):
+ response = self.client.get(f"/api/projects/{self.team.id}/signals/reports/ranking_outcomes/")
+ assert response.status_code == status.HTTP_200_OK
+ assert response.json() == {"status": expected_status, "model": None, "outcomes": []}
@parameterized.expand(
[
@@ -1052,15 +1210,15 @@ class TestSignalReportListAPI(APIBaseTest):
assert list_response.status_code == status.HTTP_200_OK
rows = list_response.json()["results"]
by_id = {row["id"]: row for row in rows}
- assert by_id[str(edited.id)]["ranking"]["stale"] is expected_stale
- assert by_id[str(other.id)]["ranking"]["stale"] is False
+ assert by_id[str(edited.id)]["ranking"]["status"] == ("stale" if expected_stale else "scored")
+ assert by_id[str(other.id)]["ranking"]["status"] == "scored"
ids = [row["id"] for row in rows]
expected_order = [str(other.id), str(edited.id)] if expected_stale else [str(edited.id), str(other.id)]
assert ids == expected_order
response = self.client.get(f"/api/projects/{self.team.id}/signals/reports/{edited.id}/")
assert response.status_code == status.HTTP_200_OK
- assert response.json()["ranking"]["stale"] is expected_stale
+ assert response.json()["ranking"]["status"] == ("stale" if expected_stale else "scored")
def test_patch_response_marks_the_score_stale(self):
self.user.is_staff = True
@@ -1072,7 +1230,7 @@ class TestSignalReportListAPI(APIBaseTest):
f"/api/projects/{self.team.id}/signals/reports/{report.id}/", data={"title": "New title"}, format="json"
)
assert response.status_code == status.HTTP_200_OK
- assert response.json()["ranking"]["stale"] is True
+ assert response.json()["ranking"]["status"] == "stale"
@parameterized.expand([("staff", True, ["ranking_score"]), ("non_staff", False, [])])
def test_artefact_routes_show_ranking_scores_to_staff_only(self, _name, is_staff, expected_types):
@@ -1563,8 +1721,8 @@ class TestSignalReportListAPI(APIBaseTest):
(str(no_priority.id), 65),
]
- @parameterized.expand([("open",), ("action",), ("fixed",)])
- def test_recommended_sort_head_variant_ranks_by_that_head_without_showing_scores_to_non_staff(self, head):
+ @parameterized.expand([("open", "open"), ("action", "action"), ("fix", "fixed")])
+ def test_recommended_sort_head_variant_ranks_by_that_head_without_showing_scores_to_non_staff(self, outcome, head):
self.user.is_staff = False
self.user.save()
low_p0 = self._create_report(title="Low P0")
@@ -1599,14 +1757,14 @@ class TestSignalReportListAPI(APIBaseTest):
)
with patch(
- "products.signals.backend.ranking.inbox_sort.posthoganalytics.get_feature_flag", return_value=head
+ "products.signals.backend.ranking.inbox_sort.posthoganalytics.get_feature_flag", return_value=outcome
) as get_flag:
response = self.client.get(self._list_url(status="ready", sort="recommended"))
assert response.status_code == status.HTTP_200_OK
assert get_flag.call_args.args[1] == self.user.distinct_id
body = response.json()
- assert body["ranker"] == head
+ assert body["ranker"] == outcome
assert [(r["id"], r["inbox_merge_key"]) for r in body["results"]] == [
(str(same_decile_p0.id), 6),
(str(high_p3.id), 9),
@@ -1614,7 +1772,7 @@ class TestSignalReportListAPI(APIBaseTest):
(str(stale_p0.id), 60),
(str(unscored_p1.id), 61),
]
- assert all(r["ranking"] is None for r in body["results"])
+ assert all(r["ranking"]["status"] == "not_available" for r in body["results"])
for raw_score in ("0.1234", "0.8123", "0.8765", "0.9876"):
assert raw_score not in response.content.decode()
diff --git a/products/signals/backend/views.py b/products/signals/backend/views.py
index 006d80eda3b8..077c02e4d5ac 100644
--- a/products/signals/backend/views.py
+++ b/products/signals/backend/views.py
@@ -1,5 +1,6 @@
import re
import json
+import math
import uuid
from collections.abc import Callable, Iterable, Sequence
from datetime import UTC, timedelta
@@ -31,6 +32,7 @@ from django.db.models import (
)
from django.db.models.expressions import OrderBy
from django.db.models.functions import Cast, Coalesce
+from django.db.models.lookups import GreaterThanOrEqual
from django.utils import timezone
import structlog
@@ -166,6 +168,8 @@ from products.signals.backend.ranking.inbox_sort import (
resolve_recommended_ranker,
score_decile,
)
+from products.signals.backend.ranking.public_outcomes import MODEL_HEAD_BY_OUTCOME, PUBLIC_OUTCOME_NAMES
+from products.signals.backend.ranking.served_outcomes import served_outcomes
from products.signals.backend.ranking.staleness import EDIT_ARTEFACT_TYPES, annotate_stale_score
from products.signals.backend.receivers import capture_verdict_reason_added_analytics
from products.signals.backend.repo_corrections import sanitized_repository
@@ -243,6 +247,8 @@ from products.signals.backend.serializers import (
PullRequestReviewCommentReactionCreateResponseSerializer,
PullRequestReviewCommentReactionCreateSerializer,
PullRequestReviewCommentUpdateSerializer,
+ RankingOutcomesResponseSerializer,
+ RankingOutcomesStatus,
ReportSignalsResponseSerializer,
SignalReportArtefactLogCreateSerializer,
SignalReportArtefactLogUpdateSerializer,
@@ -1201,8 +1207,8 @@ class SignalReportListPagination(LimitOffsetPagination):
"type": "string",
"nullable": True,
"description": (
- "The ranker that ordered the page when sort=recommended: priority, or open, action or fixed "
- "for that head of the served ranking model. Null for every other sort. Each row's "
+ "The ranker that ordered the page when sort=recommended: priority, or open, action or fix "
+ "for that outcome of the served ranking model. Null for every other sort. Each row's "
"inbox_merge_key belongs to this ranker."
),
}
@@ -1271,7 +1277,9 @@ class SignalReportViewSet(
"ranking_pr_created": "ranking_pr_created_score",
"ranking_action": "ranking_action_score",
"ranking_open": "ranking_open_score",
- "ranking_fixed": "ranking_fixed_score",
+ "ranking_fix": "ranking_fix_score",
+ # Old name of `ranking_fix`, kept for saved URLs and older clients. Not documented.
+ "ranking_fixed": "ranking_fix_score",
"ranking_discuss": "ranking_discuss_score",
"ranking_thumbs_up": "ranking_thumbs_up_score",
"ranking_reviewer_fix": "ranking_reviewer_fix_score",
@@ -1285,6 +1293,7 @@ class SignalReportViewSet(
"ranking_pr_created": "pr_created",
"ranking_action": "action",
"ranking_open": "open",
+ "ranking_fix": "fixed",
"ranking_fixed": "fixed",
"ranking_discuss": "discuss",
"ranking_thumbs_up": "thumbs_up",
@@ -1294,6 +1303,35 @@ class SignalReportViewSet(
"ranking_dismiss_lowvalue": "dismiss_lowvalue",
}
+ @extend_schema(
+ responses={200: RankingOutcomesResponseSerializer},
+ summary="List the ranking model's public outcomes",
+ description=(
+ "What each public outcome of the served ranking model predicts, with its horizon, cohort, base rate "
+ "and readability from the model that scored this project's reports. Users who cannot see the ranking "
+ "model get status not_available and no outcomes."
+ ),
+ operation_id="signals_reports_ranking_outcomes_list",
+ )
+ @action(
+ detail=False,
+ methods=["get"],
+ url_path="ranking_outcomes",
+ required_scopes=["task:read"],
+ pagination_class=None,
+ )
+ def ranking_outcomes(self, request: Request, **kwargs) -> Response:
+ model: str | None = None
+ outcomes: list[dict[str, Any]] = []
+ if not self._can_view_inbox_ranking:
+ status_value = RankingOutcomesStatus.NOT_AVAILABLE
+ else:
+ model, outcomes = served_outcomes(self.team_id)
+ status_value = RankingOutcomesStatus.SCORED if model is not None else RankingOutcomesStatus.UNSCORED
+ return Response(
+ RankingOutcomesResponseSerializer({"status": status_value.value, "model": model, "outcomes": outcomes}).data
+ )
+
@extend_schema(request=ReportReadStateRequestSerializer, responses=ReportReadStateResponseSerializer)
@action(detail=False, methods=["post"], required_scopes=["task:read"])
def read_state(self, request, **kwargs):
@@ -1383,6 +1421,7 @@ class SignalReportViewSet(
qs = self._apply_signal_report_priority_filter(qs)
if self.action == "list":
qs = self._annotate_signal_report_ranking(qs)
+ qs = self._apply_signal_report_ranking_filters(qs)
qs = self._prefetch_signal_report_priority_artefacts(qs)
if self.action != "bulk_state":
# `bulk_state` answers with one outcome per id, never a serialized report, and the list
@@ -1955,62 +1994,112 @@ class SignalReportViewSet(
# `ordering=ranking_<head>` sorts by the served model's probability for that head, read from
# the latest ranking_score artefact as `results -> <served_key> -> scores -> <head>`. Only
# the heads the ordering names are annotated, because each one is a correlated subquery.
- # A value that is not a JSON number reads as NULL, so one bad row cannot fail the list.
- # The guard is in the CASE, not the filter: the latest artefact decides, as it does for the
- # `ranking` field, so a bad latest row makes the report unscored rather than older-scored.
- # A stale score (the report was edited after the text it scored) also sorts as unscored.
ordered_fields = {clause.lstrip("-") for clause in self._parse_signal_report_ordering()}
+ annotated: set[str] = set()
for field, head in self._RANKING_ORDERING_HEADS.items():
annotation = self._SIGNAL_REPORT_ORDERING_FIELDS[field]
- if annotation not in ordered_fields:
+ # An alias shares its annotation with the field it renames, so annotate each one once.
+ if annotation not in ordered_fields or annotation in annotated:
continue
- content = Cast(F("content"), output_field=JSONField())
- served_key = Func(
- content, Value("served_key"), function="jsonb_extract_path_text", output_field=CharField()
+ annotated.add(annotation)
+ queryset = queryset.annotate(**{annotation: self._latest_ranking_value(head, "scores")})
+ return queryset
+
+ @staticmethod
+ def _latest_ranking_value(head: str, key: Literal["scores", "lifts"]) -> Subquery:
+ # The served model's value for one head from the latest ranking_score artefact, as
+ # `results -> <served_key> -> <key> -> <head>`. A value that is not a JSON number reads as NULL,
+ # so one bad row cannot fail the list. The guard is in the CASE, not the filter: the latest
+ # artefact decides, as it does for the `ranking` field, so a bad latest row makes the report
+ # unscored rather than older-scored. A stale score (the report was edited after the text it
+ # scored) also reads as NULL.
+ content = Cast(F("content"), output_field=JSONField())
+ served_key = Func(content, Value("served_key"), function="jsonb_extract_path_text", output_field=CharField())
+ return Subquery(
+ annotate_stale_score(
+ SignalReportArtefact.objects.filter(
+ report_id=OuterRef("id"),
+ type=SignalReportArtefact.ArtefactType.RANKING_SCORE,
+ ).order_by("-created_at")
)
- latest_score = Subquery(
- annotate_stale_score(
- SignalReportArtefact.objects.filter(
- report_id=OuterRef("id"),
- type=SignalReportArtefact.ArtefactType.RANKING_SCORE,
- ).order_by("-created_at")
- )
- .annotate(
- _score=Case(
- When(
- content__startswith="{",
- then=Func(
- content,
- Value("results"),
- served_key,
- Value("scores"),
- Value(head),
- function="jsonb_extract_path",
- output_field=JSONField(),
- ),
+ .annotate(
+ _score=Case(
+ When(
+ content__startswith="{",
+ then=Func(
+ content,
+ Value("results"),
+ served_key,
+ Value(key),
+ Value(head),
+ function="jsonb_extract_path",
+ output_field=JSONField(),
),
- default=Value(None),
- output_field=JSONField(),
),
+ default=Value(None),
+ output_field=JSONField(),
+ ),
+ )
+ .annotate(_score_type=Func(F("_score"), function="jsonb_typeof", output_field=CharField()))
+ .annotate(
+ _score_value=Case(
+ When(
+ _score_type="number",
+ _score_is_stale=False,
+ then=Cast(F("_score"), output_field=FloatField()),
+ ),
+ default=Value(None),
+ output_field=FloatField(),
)
- .annotate(_score_type=Func(F("_score"), function="jsonb_typeof", output_field=CharField()))
- .annotate(
- _score_value=Case(
- When(
- _score_type="number",
- _score_is_stale=False,
- then=Cast(F("_score"), output_field=FloatField()),
- ),
- default=Value(None),
- output_field=FloatField(),
- )
- )
- .values("_score_value")[:1],
- output_field=FloatField(),
)
- queryset = queryset.annotate(**{annotation: latest_score})
+ .values("_score_value")[:1],
+ output_field=FloatField(),
+ )
+
+ def _apply_signal_report_ranking_filters(self, queryset):
+ # Each named head is one correlated subquery, compared with `>=`. NULL never matches, so
+ # unscored and stale reports drop out of every ranking filter.
+ filters: tuple[tuple[str, Literal["scores", "lifts"]], ...] = (
+ ("ranking_min_lift", "lifts"),
+ ("ranking_min_probability", "scores"),
+ )
+ for param, key in filters:
+ minimums = self._parse_ranking_minimums(param)
+ if not minimums:
+ continue
+ for outcome, minimum in minimums.items():
+ head = MODEL_HEAD_BY_OUTCOME[outcome]
+ queryset = queryset.filter(GreaterThanOrEqual(self._latest_ranking_value(head, key), minimum))
return queryset
+ def _parse_ranking_minimums(self, param: str) -> dict[str, float]:
+ raw = self.request.query_params.get(param)
+ if raw is None or not raw.strip():
+ return {}
+ if not self._can_view_inbox_ranking:
+ # The serializer hides ranking scores from these users, and the filter would leak them.
+ raise serializers.ValidationError({param: f"Filtering by {param} is not available to you."})
+ valid = ", ".join(PUBLIC_OUTCOME_NAMES)
+ minimums: dict[str, float] = {}
+ for pair in raw.split(","):
+ outcome, sep, value = pair.strip().partition(":")
+ outcome = outcome.strip()
+ if outcome not in PUBLIC_OUTCOME_NAMES:
+ raise serializers.ValidationError({param: f"Unknown outcome {outcome!r}. Valid outcomes: {valid}."})
+ try:
+ minimum = float(value) if sep else math.nan
+ except ValueError:
+ minimum = math.nan
+ if not math.isfinite(minimum) or minimum < 0:
+ raise serializers.ValidationError(
+ {
+ param: f"Use <outcome>:<number> pairs with a number of 0 or more, e.g. fix:2. Valid outcomes: {valid}."
+ }
+ )
+ # A report must meet every pair, so a repeated outcome keeps its strictest minimum.
+ minimums[outcome] = max(minimum, minimums.get(outcome, minimum))
+ return minimums
+
def _needs_priority_annotation(self) -> bool:
# The priority value is a correlated subquery too, and the serializer renders priority from
# the prefetched artefacts instead, so only a priority filter or a priority sort needs it.
@@ -2574,7 +2663,7 @@ class SignalReportViewSet(
description=(
"Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when "
"ordering is supplied. recommended resolves per user to a ranker: the priority order, or "
- "the open, action or fixed head of the served ranking model with unscored and stale reports "
+ "the open, action or fix outcome of the served ranking model with unscored and stale reports "
"in priority order after the scored ones. The response's ranker field names the one used."
),
),
@@ -2587,12 +2676,37 @@ class SignalReportViewSet(
"Comma-separated ordering clauses. Each clause is a field name optionally prefixed with '-' "
"for descending. Allowed fields: status, is_suggested_reviewer, signal_count, total_weight, "
"priority, created_at, updated_at, id, ranking_pr_merged, ranking_pr_created, ranking_action, "
- "ranking_open, ranking_fixed, ranking_discuss, ranking_thumbs_up, ranking_reviewer_fix, "
+ "ranking_open, ranking_fix, ranking_discuss, ranking_thumbs_up, ranking_reviewer_fix, "
"ranking_refund, ranking_dismiss_wrong, ranking_dismiss_lowvalue. Defaults to '-is_suggested_reviewer,status,-updated_at'. The ranking_* fields "
"sort by the served ranking model's probability for that outcome head, with unscored reports "
"last in either direction. They need staff or the inbox model sort rollout: other users get a 400."
),
),
+ OpenApiParameter(
+ name="ranking_min_lift",
+ type=OpenApiTypes.STR,
+ location=OpenApiParameter.QUERY,
+ required=False,
+ description=(
+ "Comma-separated <outcome>:<lift> pairs, e.g. fix:2,open:1.5. Keeps reports whose served "
+ "ranking lift is at least the value for every named outcome. Outcomes: open, action, fix. Read "
+ "inbox-ranking-outcomes-list for what each outcome means. Unscored and stale reports never match. "
+ "Needs staff or the inbox model sort rollout: other users get a 400."
+ ),
+ ),
+ OpenApiParameter(
+ name="ranking_min_probability",
+ type=OpenApiTypes.STR,
+ location=OpenApiParameter.QUERY,
+ required=False,
+ description=(
+ "Comma-separated <outcome>:<probability> pairs, e.g. fix:0.3. Keeps reports whose served "
+ "ranking probability is at least the value for every named outcome. Outcomes: open, action, "
+ "fix. Read inbox-ranking-outcomes-list for what each outcome means. Unscored and stale reports "
+ "never match. "
+ "Needs staff or the inbox model sort rollout: other users get a 400."
+ ),
+ ),
OpenApiParameter(
name="created_after",
type=OpenApiTypes.DATETIME,
diff --git a/products/signals/frontend/generated/api.schemas.ts b/products/signals/frontend/generated/api.schemas.ts
index f89a196c5ada..adc9a1f59643 100644
--- a/products/signals/frontend/generated/api.schemas.ts
+++ b/products/signals/frontend/generated/api.schemas.ts
@@ -1417,34 +1417,71 @@ export const SignalReportBillingExemptReasonEnumApi = {
} as const
/**
- * Outcome head name to its calibrated probability. Empty when the served model skipped the report, and always empty for non-staff users.
+ * * `scored` - scored
+ * * `stale` - stale
+ * * `unscored` - unscored
+ * * `not_available` - not_available
*/
-export type ReportRankingApiScores = { [key: string]: number }
+export type RankingStatusEnumApi = (typeof RankingStatusEnumApi)[keyof typeof RankingStatusEnumApi]
+
+export const RankingStatusEnumApi = {
+ Scored: 'scored',
+ Stale: 'stale',
+ Unscored: 'unscored',
+ NotAvailable: 'not_available',
+} as const
/**
- * Outcome head name to its probability divided by the head's training base rate, e.g. 2.7 means 2.7x as likely as the average report. A head without a saved base rate has no entry.
+ * * `open` - Open
+ * * `action` - Action
+ * * `fix` - Fix
*/
-export type ReportRankingApiLifts = { [key: string]: number }
+export type RankingOutcomeEnumApi = (typeof RankingOutcomeEnumApi)[keyof typeof RankingOutcomeEnumApi]
+
+export const RankingOutcomeEnumApi = {
+ Open: 'open',
+ Action: 'action',
+ Fix: 'fix',
+} as const
+
+export interface ReportRankingPredictionApi {
+ /** Outcome the model predicts. Read inbox-ranking-outcomes-list for what each outcome means.
+ *
+ * * `open` - Open
+ * * `action` - Action
+ * * `fix` - Fix */
+ outcome: RankingOutcomeEnumApi
+ /** The model's calibrated probability that the outcome happens within the outcome's horizon. */
+ probability: number
+ /**
+ * The probability divided by the outcome's base rate. 1.0 is an average report, 2.7 means 2.7x as likely as the average report. Compare lifts across outcomes, not raw probabilities. Null when the model has no saved base rate for the outcome.
+ * @nullable
+ */
+ lift: number | null
+ /** True when the training run could read the outcome's holdout AUC. Treat other outcomes with caution. */
+ readable: boolean
+}
export interface ReportRankingApi {
- /** Key of the served model in the scoring pass, as `<model_name>@<model_version>`. Empty for non-staff users. */
- served_key: string
- /** Feature family of the served model. Empty for non-staff users. */
- model_name: string
- /** Training partition of the served model, as `YYYY-MM-DD`. Empty for non-staff users. */
- model_version: string
- /** Version of the serving manifest that chose the model. Empty for non-staff users. */
- manifest_version: string
- /** When the scoring sweep scored the report. */
- scored_at: string
- /** Outcome head name to its calibrated probability. Empty when the served model skipped the report, and always empty for non-staff users. */
- scores: ReportRankingApiScores
- /** Outcome head name to its probability divided by the head's training base rate, e.g. 2.7 means 2.7x as likely as the average report. A head without a saved base rate has no entry. */
- lifts: ReportRankingApiLifts
- /** Heads whose holdout AUC the training run could read. Treat scores of other heads with caution. */
- readable_heads: string[]
- /** True when the report's title or summary was edited after the text this score read. The score describes the old text: the inbox hides its lift and the model sort treats the report as unscored. */
- stale: boolean
+ /** scored: the predictions describe the current text. stale: the title or summary changed after the text the model read, so the predictions describe the old text and the model sorts and filters treat the report as unscored. unscored: the model has not scored the report yet, or skipped it. This does not mean a low score. not_available: you cannot see the ranking model's output, so every other field is empty.
+ *
+ * * `scored` - scored
+ * * `stale` - stale
+ * * `unscored` - unscored
+ * * `not_available` - not_available */
+ status: RankingStatusEnumApi
+ /**
+ * Key of the served model that wrote the score, as `<model_name>@<model_version>`. Do not compare predictions from different keys.
+ * @nullable
+ */
+ model: string | null
+ /**
+ * When the scoring sweep scored the report.
+ * @nullable
+ */
+ scored_at: string | null
+ /** The served model's predictions, one per public outcome that it scored. A prediction says what may happen, never what did: the report's status and its PR fields say whether it was resolved. */
+ predictions: ReportRankingPredictionApi[]
}
export interface ReportPlanStepApi {
@@ -1635,12 +1672,12 @@ export interface SignalReportListApi {
* @nullable
*/
readonly channel_id: string | null
- /** The served model's score from the latest ranking score artefact. Null for users who are neither staff nor in the inbox model sort rollout, and null when the report has no score. */
- readonly ranking: ReportRankingApi | null
+ /** The served ranking model's predictions from the latest ranking score artefact. Its status says whether the report is scored, stale or unscored, or that the user cannot see the ranking model. */
+ readonly ranking: ReportRankingApi
/** The plan this report leads or is a step of, with every step in it. A report that other reports link to with `part_of` is a plan. Null when the report is in no plan, and always null in list responses, because finding a plan's steps scans the project's report links. */
readonly plan: ReportPlanApi | null
/**
- * Coarse sort key from the ranker named by the list response's ranker field, for sort=recommended only. Lower keys rank first. It is the score decile of the ranker's head (open, action or fixed), then the priority as a tiebreak. The priority ranker uses the priority only. Reports without a usable score sort after scored ones. Null for every other sort.
+ * Coarse sort key from the ranker named by the list response's ranker field, for sort=recommended only. Lower keys rank first. It is the score decile of the ranker's outcome (open, action or fix), then the priority as a tiebreak. The priority ranker uses the priority only. Reports without a usable score sort after scored ones. Null for every other sort.
* @nullable
*/
readonly inbox_merge_key: number | null
@@ -1654,7 +1691,7 @@ export interface PaginatedSignalReportListListApi {
previous?: string | null
results: SignalReportListApi[]
/**
- * The ranker that ordered the page when sort=recommended: priority, or open, action or fixed for that head of the served ranking model. Null for every other sort. Each row's inbox_merge_key belongs to this ranker.
+ * The ranker that ordered the page when sort=recommended: priority, or open, action or fix for that outcome of the served ranking model. Null for every other sort. Each row's inbox_merge_key belongs to this ranker.
* @nullable
*/
ranker?: string | null
@@ -1885,8 +1922,8 @@ export interface SignalReportApi {
* @nullable
*/
readonly channel_id: string | null
- /** The served model's score from the latest ranking score artefact. Null for users who are neither staff nor in the inbox model sort rollout, and null when the report has no score. */
- readonly ranking: ReportRankingApi | null
+ /** The served ranking model's predictions from the latest ranking score artefact. Its status says whether the report is scored, stale or unscored, or that the user cannot see the ranking model. */
+ readonly ranking: ReportRankingApi
/** The plan this report leads or is a step of, with every step in it. A report that other reports link to with `part_of` is a plan. Null when the report is in no plan, and always null in list responses, because finding a plan's steps scans the project's report links. */
readonly plan: ReportPlanApi | null
}
@@ -5081,6 +5118,66 @@ export interface PullRequestCiStatusesResponseApi {
readonly statuses: readonly PullRequestCiStatusApi[]
}
+/**
+ * * `scored` - scored
+ * * `unscored` - unscored
+ * * `not_available` - not_available
+ */
+export type RankingOutcomesStatusEnumApi =
+ (typeof RankingOutcomesStatusEnumApi)[keyof typeof RankingOutcomesStatusEnumApi]
+
+export const RankingOutcomesStatusEnumApi = {
+ Scored: 'scored',
+ Unscored: 'unscored',
+ NotAvailable: 'not_available',
+} as const
+
+export interface RankingOutcomeInfoApi {
+ /** Name of the outcome in the `ranking` field's predictions.
+ *
+ * * `open` - Open
+ * * `action` - Action
+ * * `fix` - Fix */
+ outcome: RankingOutcomeEnumApi
+ /** Short label the inbox shows for the outcome. */
+ label: string
+ /** What the outcome predicts, in one sentence. */
+ description: string
+ /**
+ * Days after scoring within which the outcome counts. Null when the served model did not save it.
+ * @nullable
+ */
+ horizon_days: number | null
+ /**
+ * Name of the report set the outcome was trained on. Null when the served model did not save it.
+ * @nullable
+ */
+ cohort: string | null
+ /**
+ * The outcome's base rate: a report's lift is its probability divided by this value. Null when the served model has no saved base rate for the outcome.
+ * @nullable
+ */
+ base_rate: number | null
+ /** True when the training run could read the outcome's holdout AUC. Treat other outcomes with caution. */
+ readable: boolean
+}
+
+export interface RankingOutcomesResponseApi {
+ /** scored: the served model has scored reports in this project. unscored: no readable score yet. not_available: you cannot see the ranking model's output.
+ *
+ * * `scored` - scored
+ * * `unscored` - unscored
+ * * `not_available` - not_available */
+ status: RankingOutcomesStatusEnumApi
+ /**
+ * Key of the served model, as `<model_name>@<model_version>`, from the project's latest score.
+ * @nullable
+ */
+ model: string | null
+ /** One entry per public outcome, in the inbox order. */
+ outcomes: RankingOutcomeInfoApi[]
+}
+
export interface ReportReadStateRequestApi {
/**
* Reports to read or update, limited to the current project.
@@ -9464,13 +9561,21 @@ export type SignalsReportsListParams = {
*/
offset?: number
/**
- * Comma-separated ordering clauses. Each clause is a field name optionally prefixed with '-' for descending. Allowed fields: status, is_suggested_reviewer, signal_count, total_weight, priority, created_at, updated_at, id, ranking_pr_merged, ranking_pr_created, ranking_action, ranking_open, ranking_fixed, ranking_discuss, ranking_thumbs_up, ranking_reviewer_fix, ranking_refund, ranking_dismiss_wrong, ranking_dismiss_lowvalue. Defaults to '-is_suggested_reviewer,status,-updated_at'. The ranking_* fields sort by the served ranking model's probability for that outcome head, with unscored reports last in either direction. They need staff or the inbox model sort rollout: other users get a 400.
+ * Comma-separated ordering clauses. Each clause is a field name optionally prefixed with '-' for descending. Allowed fields: status, is_suggested_reviewer, signal_count, total_weight, priority, created_at, updated_at, id, ranking_pr_merged, ranking_pr_created, ranking_action, ranking_open, ranking_fix, ranking_discuss, ranking_thumbs_up, ranking_reviewer_fix, ranking_refund, ranking_dismiss_wrong, ranking_dismiss_lowvalue. Defaults to '-is_suggested_reviewer,status,-updated_at'. The ranking_* fields sort by the served ranking model's probability for that outcome head, with unscored reports last in either direction. They need staff or the inbox model sort rollout: other users get a 400.
*/
ordering?: string
/**
* Comma-separated list of priorities to include. Valid values: P0, P1, P2, P3, P4. Reports without a priority assignment are excluded when this filter is set.
*/
priority?: string
+ /**
+ * Comma-separated <outcome>:<lift> pairs, e.g. fix:2,open:1.5. Keeps reports whose served ranking lift is at least the value for every named outcome. Outcomes: open, action, fix. Read inbox-ranking-outcomes-list for what each outcome means. Unscored and stale reports never match. Needs staff or the inbox model sort rollout: other users get a 400.
+ */
+ ranking_min_lift?: string
+ /**
+ * Comma-separated <outcome>:<probability> pairs, e.g. fix:0.3. Keeps reports whose served ranking probability is at least the value for every named outcome. Outcomes: open, action, fix. Read inbox-ranking-outcomes-list for what each outcome means. Unscored and stale reports never match. Needs staff or the inbox model sort rollout: other users get a 400.
+ */
+ ranking_min_probability?: string
/**
* Reviewer scope: for_me, entire_project, or teammate. Pass teammate_uuid with teammate.
*/
@@ -9488,7 +9593,7 @@ export type SignalsReportsListParams = {
*/
search?: string
/**
- * Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fixed head of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used.
+ * Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used.
*/
sort?: string
/**
diff --git a/products/signals/frontend/generated/api.ts b/products/signals/frontend/generated/api.ts
index 87056e4bb24f..e1dd8c8a2443 100644
--- a/products/signals/frontend/generated/api.ts
+++ b/products/signals/frontend/generated/api.ts
@@ -49,6 +49,7 @@ import type {
PullRequestReviewCommentCreateResponseApi,
PullRequestReviewCommentReactionCreateApi,
PullRequestReviewCommentReactionCreateResponseApi,
+ RankingOutcomesResponseApi,
RecordCheckResultRequestApi,
RecordCheckResultResponseApi,
RecordStructuredOutputRequestApi,
@@ -1464,6 +1465,24 @@ export const signalsReportsPrCiStatuses = async (
})
}
+export const getSignalsReportsRankingOutcomesListUrl = (projectId: string) => {
+ return `/api/projects/${projectId}/signals/reports/ranking_outcomes/`
+}
+
+/**
+ * What each public outcome of the served ranking model predicts, with its horizon, cohort, base rate and readability from the model that scored this project's reports. Users who cannot see the ranking model get status not_available and no outcomes.
+ * @summary List the ranking model's public outcomes
+ */
+export const signalsReportsRankingOutcomesList = async (
+ projectId: string,
+ options?: RequestInit
+): Promise<RankingOutcomesResponseApi> => {
+ return apiMutator<RankingOutcomesResponseApi>(getSignalsReportsRankingOutcomesListUrl(projectId), {
+ ...options,
+ method: 'GET',
+ })
+}
+
export const getSignalsReportsReadStateCreateUrl = (projectId: string) => {
return `/api/projects/${projectId}/signals/reports/read_state/`
}
diff --git a/products/signals/frontend/inbox/components/cards/InboxCards.stories.tsx b/products/signals/frontend/inbox/components/cards/InboxCards.stories.tsx
index e392faea9797..77f096632087 100644
--- a/products/signals/frontend/inbox/components/cards/InboxCards.stories.tsx
+++ b/products/signals/frontend/inbox/components/cards/InboxCards.stories.tsx
@@ -5,6 +5,8 @@ import { FEATURE_FLAGS } from 'lib/constants'
import { NodeKind, type InsightVizNode, type TrendsQuery } from '~/queries/schema/schema-general'
import { BaseMathType, ChartDisplayType, PropertyMathType } from '~/types'
+import { RankingOutcomeEnumApi } from 'products/signals/frontend/generated/api.schemas'
+
import { userEvent } from 'storybook/test'
import { makeReport, pullRequestReports, reportTabReports, runReportsMany } from '../../__mocks__/inboxMocks'
@@ -257,14 +259,22 @@ export const ReportCardWithImpact: Story = {
),
}
-const RANKING = {
- served_key: 'report_embeddings@2026-06-10',
- model_name: 'report_embeddings',
- model_version: '2026-06-10',
- manifest_version: '12',
- scored_at: '2026-06-10T09:00:00Z',
- readable_heads: ['open', 'action', 'fixed', 'pr_merged'],
- stale: false,
+function ranking(
+ scores: Partial<Record<RankingOutcomeEnumApi, number>>,
+ lifts: Partial<Record<RankingOutcomeEnumApi, number>>,
+ stale = false
+): SignalReport['ranking'] {
+ return {
+ status: stale ? 'stale' : 'scored',
+ model: 'report_embeddings@2026-06-10',
+ scored_at: '2026-06-10T09:00:00Z',
+ predictions: Object.values(RankingOutcomeEnumApi).flatMap((outcome) => {
+ const probability = scores[outcome]
+ return probability === undefined
+ ? []
+ : [{ outcome, probability, lift: lifts[outcome] ?? null, readable: true }]
+ }),
+ }
}
// Under the Action sort: a lift above 1x, a lift below 1x, a probability with no lift, an unscored report, and a report edited after scoring.
@@ -273,29 +283,25 @@ const rankedReports: SignalReport[] = [
title: 'Checkout button does nothing on Safari',
priority: 'P1',
source_products: ['session_replay'],
- ranking: {
- ...RANKING,
- scores: { open: 0.91, action: 0.78, fixed: 0.33, pr_merged: 0.52 },
- lifts: { open: 0.6, action: 2.1, fixed: 6.0, pr_merged: 2.6 },
- },
+ ranking: ranking({ open: 0.91, action: 0.78, fix: 0.33 }, { open: 0.6, action: 2.1, fix: 6.0 }),
}),
makeReport({
title: 'Tooltip flickers on the funnel chart',
priority: 'P3',
source_products: ['session_replay'],
- ranking: { ...RANKING, scores: { action: 0.21 }, lifts: { action: 0.6 } },
+ ranking: ranking({ action: 0.21 }, { action: 0.6 }),
}),
makeReport({
title: 'Slow dashboard load for large projects',
priority: 'P2',
source_products: ['error_tracking'],
- ranking: { ...RANKING, scores: { open: 0.4, action: 0.062, fixed: 0.05 }, lifts: {} },
+ ranking: ranking({ open: 0.4, action: 0.062, fix: 0.05 }, {}),
}),
makeReport({ title: 'Typo in the billing settings page', priority: 'P4', ranking: null }),
makeReport({
title: 'Retry loop in the export job',
priority: 'P3',
- ranking: { ...RANKING, scores: { action: 0.5 }, lifts: { action: 1.2 }, stale: true },
+ ranking: ranking({ action: 0.5 }, { action: 1.2 }, true),
}),
]
diff --git a/products/signals/frontend/inbox/components/cards/ReportCard.test.tsx b/products/signals/frontend/inbox/components/cards/ReportCard.test.tsx
index e9e556f6a97d..f43bb66fbd01 100644
--- a/products/signals/frontend/inbox/components/cards/ReportCard.test.tsx
+++ b/products/signals/frontend/inbox/components/cards/ReportCard.test.tsx
@@ -422,7 +422,7 @@ describe('ReportCard', () => {
})
it.each([
- ['with the impact column', true, 'ranking_fixed', '2.7x more likely'],
+ ['with the impact column', true, 'ranking_fix', '2.7x more likely'],
[
'without the impact column, falling back to the probability for a head with no lift',
false,
@@ -436,15 +436,13 @@ describe('ReportCard', () => {
const report = makeReport('r-2', {
metrics: [makeMetric()],
ranking: {
- served_key: 'report_embeddings@2026-09-30',
- model_name: 'report_embeddings',
- model_version: '2026-09-30',
- manifest_version: 'manifest',
+ status: 'scored',
+ model: 'report_embeddings@2026-09-30',
scored_at: '2026-09-30T12:00:00Z',
- scores: { fixed: 0.41, action: 0.062 },
- lifts: { fixed: 2.7 },
- readable_heads: ['action', 'fixed'],
- stale: false,
+ predictions: [
+ { outcome: 'action', probability: 0.062, lift: null, readable: true },
+ { outcome: 'fix', probability: 0.41, lift: 2.7, readable: true },
+ ],
},
})
const { container } = render(<ReportCard report={report} rankingSortField={sortField} />)
@@ -459,23 +457,18 @@ describe('ReportCard', () => {
// The harness renders a card for every test; these assert against their own.
cleanup()
enableImpactColumn()
- const { rerender } = render(<ReportCard report={makeReport('r-2')} rankingSortField="ranking_fixed" />)
+ const { rerender } = render(<ReportCard report={makeReport('r-2')} rankingSortField="ranking_fix" />)
expect(screen.getByText('Not scored yet')).toBeInTheDocument()
const stale = makeReport('r-2', {
ranking: {
- served_key: 'report_embeddings@2026-09-30',
- model_name: 'report_embeddings',
- model_version: '2026-09-30',
- manifest_version: 'manifest',
+ status: 'stale',
+ model: 'report_embeddings@2026-09-30',
scored_at: '2026-09-30T12:00:00Z',
- scores: { fixed: 0.41 },
- lifts: { fixed: 2.7 },
- readable_heads: ['fixed'],
- stale: true,
+ predictions: [{ outcome: 'fix', probability: 0.41, lift: 2.7, readable: true }],
},
})
- rerender(<ReportCard report={stale} rankingSortField="ranking_fixed" />)
+ rerender(<ReportCard report={stale} rankingSortField="ranking_fix" />)
expect(screen.getByText('Score out of date')).toBeInTheDocument()
expect(screen.queryByText('2.7x more likely')).not.toBeInTheDocument()
diff --git a/products/signals/frontend/inbox/components/cards/ReportCardRankingTag.tsx b/products/signals/frontend/inbox/components/cards/ReportCardRankingTag.tsx
index bbd05b19f981..ebdf27cd61c1 100644
--- a/products/signals/frontend/inbox/components/cards/ReportCardRankingTag.tsx
+++ b/products/signals/frontend/inbox/components/cards/ReportCardRankingTag.tsx
@@ -1,12 +1,16 @@
import clsx from 'clsx'
+import { useValues } from 'kea'
import { useCallback } from 'react'
import { LemonTag, LemonTagType, Tooltip } from '@posthog/lemon-ui'
+import { userLogic } from 'scenes/userLogic'
+
import { INBOX_RANKING_SCORES, InboxRankingScore, rankingScoreForField } from '../../filterOptions'
import { captureInboxRankingDetailsViewed } from '../../inboxAnalytics'
import type { InboxRankingSortField } from '../../logics/inboxFiltersLogic'
import { SignalReport } from '../../types'
+import { rankingPrediction } from '../../utils/reportOrdering'
import {
formatRankingLift,
formatRankingLiftComparison,
@@ -26,24 +30,33 @@ const STRENGTH_TAG_TYPES: Record<RankingStrength, LemonTagType> = {
unlikely: 'muted',
}
-function rankingLift(ranking: Ranking, head: string): number | null {
- const lift = ranking.lifts?.[head]
+function rankingLift(ranking: Ranking, outcome: string): number | null {
+ const lift = rankingPrediction(ranking, outcome)?.lift
return typeof lift === 'number' && Number.isFinite(lift) ? lift : null
}
-/** Raw probabilities and the model version come only to staff. Other users get lifts alone. */
-function rankingProbability(ranking: Ranking, head: string): number | null {
- const probability = ranking.scores[head]
+/** The inbox shows raw probabilities and the model key to staff only. Other users see lifts alone. */
+function rankingProbability(ranking: Ranking, outcome: string, isStaff: boolean): number | null {
+ const probability = isStaff ? rankingPrediction(ranking, outcome)?.probability : undefined
return typeof probability === 'number' ? probability : null
}
-function RankingHoverCard({ ranking, active }: { ranking: Ranking; active: InboxRankingScore }): JSX.Element {
- const lift = rankingLift(ranking, active.head)
+function RankingHoverCard({
+ ranking,
+ active,
+ isStaff,
+}: {
+ ranking: Ranking
+ active: InboxRankingScore
+ isStaff: boolean
+}): JSX.Element {
+ const lift = rankingLift(ranking, active.outcome)
const others = INBOX_RANKING_SCORES.filter(
(score) =>
score !== active &&
- ranking.readable_heads.includes(score.head) &&
- (rankingLift(ranking, score.head) !== null || rankingProbability(ranking, score.head) !== null)
+ rankingPrediction(ranking, score.outcome)?.readable &&
+ (rankingLift(ranking, score.outcome) !== null ||
+ rankingProbability(ranking, score.outcome, isStaff) !== null)
)
return (
<div className="flex max-w-96 min-w-80 flex-col gap-2 text-xs">
@@ -61,11 +74,11 @@ function RankingHoverCard({ ranking, active }: { ranking: Ranking; active: Inbox
)}
<div className="flex flex-col gap-1">
{[active, ...others].map((score) => {
- const scoreLift = rankingLift(ranking, score.head)
- const probability = rankingProbability(ranking, score.head)
+ const scoreLift = rankingLift(ranking, score.outcome)
+ const probability = rankingProbability(ranking, score.outcome, isStaff)
return (
<div
- key={score.head}
+ key={score.outcome}
className={clsx('flex items-center gap-2', score === active && 'font-bold')}
>
<span className="w-16 shrink-0">{score.name}</span>
@@ -90,10 +103,8 @@ function RankingHoverCard({ ranking, active }: { ranking: Ranking; active: Inbox
<div className="opacity-70">
Very likely means 2x the average or more. The model estimates this from past reports.
</div>
- {ranking.model_name && (
- <div className="border-t border-primary pt-1 font-mono opacity-70">
- {ranking.model_name}@{ranking.model_version}
- </div>
+ {isStaff && ranking.model && (
+ <div className="border-t border-primary pt-1 font-mono opacity-70">{ranking.model}</div>
)}
</div>
)
@@ -111,10 +122,19 @@ export function ReportCardRankingTag({
sortField: InboxRankingSortField
}): JSX.Element {
const score = rankingScoreForField(sortField)
+ const isStaff = useValues(userLogic).user?.is_staff ?? false
const ranking = report.ranking
- const lift = ranking ? rankingLift(ranking, score.head) : null
- const probability = ranking ? rankingProbability(ranking, score.head) : null
- const state = !ranking || (lift === null && probability === null) ? 'unscored' : ranking.stale ? 'stale' : 'scored'
+ const lift = ranking ? rankingLift(ranking, score.outcome) : null
+ const probability = ranking ? rankingProbability(ranking, score.outcome, isStaff) : null
+ const state =
+ !ranking ||
+ ranking.status === 'unscored' ||
+ ranking.status === 'not_available' ||
+ (lift === null && probability === null)
+ ? 'unscored'
+ : ranking.status === 'stale'
+ ? 'stale'
+ : 'scored'
const strength = lift !== null ? rankingStrength(lift) : null
// Tooltip re-fires onOpen when the callback identity changes, so a new one each render would double count views.
const onOpen = useCallback(
@@ -153,7 +173,11 @@ export function ReportCardRankingTag({
)
}
return (
- <Tooltip title={<RankingHoverCard ranking={ranking} active={score} />} onOpen={onOpen} openOnClick>
+ <Tooltip
+ title={<RankingHoverCard ranking={ranking} active={score} isStaff={isStaff} />}
+ onOpen={onOpen}
+ openOnClick
+ >
<LemonTag
size="small"
type={strength ? STRENGTH_TAG_TYPES[strength] : 'default'}
diff --git a/products/signals/frontend/inbox/filterOptions.tsx b/products/signals/frontend/inbox/filterOptions.tsx
index 70ae65ba5228..20fdb780f19f 100644
--- a/products/signals/frontend/inbox/filterOptions.tsx
+++ b/products/signals/frontend/inbox/filterOptions.tsx
@@ -99,6 +99,8 @@ export const INBOX_SORT_OPTIONS: InboxSortOption[] = [
*/
export interface InboxRankingScore {
field: InboxRankingSortField
+ outcome: string
+ /** The model head in raw `ranking_score` artefacts. It differs from the outcome for `fix`, which reads `fixed`. */
head: string
name: string
sortLabel: string
@@ -110,6 +112,7 @@ export interface InboxRankingScore {
export const INBOX_RANKING_SCORES: InboxRankingScore[] = [
{
field: 'ranking_open',
+ outcome: 'open',
head: 'open',
name: 'Open',
sortLabel: 'Most likely to be opened',
@@ -118,6 +121,7 @@ export const INBOX_RANKING_SCORES: InboxRankingScore[] = [
},
{
field: 'ranking_action',
+ outcome: 'action',
head: 'action',
name: 'Action',
sortLabel: 'Most likely to be acted on',
@@ -125,7 +129,8 @@ export const INBOX_RANKING_SCORES: InboxRankingScore[] = [
description: 'A person or their agent acts on the report within 14 days.',
},
{
- field: 'ranking_fixed',
+ field: 'ranking_fix',
+ outcome: 'fix',
head: 'fixed',
name: 'Fix',
sortLabel: 'Most likely to be fixed',
diff --git a/products/signals/frontend/inbox/inboxAnalytics.test.ts b/products/signals/frontend/inbox/inboxAnalytics.test.ts
index 365b0a8f5ad2..2341e257d0b4 100644
--- a/products/signals/frontend/inbox/inboxAnalytics.test.ts
+++ b/products/signals/frontend/inbox/inboxAnalytics.test.ts
@@ -123,15 +123,10 @@ describe('inboxAnalytics', () => {
signal_count: 3,
is_suggested_reviewer: true,
ranking: {
- served_key: 'report_embeddings@2026-09-30',
- model_name: 'report_embeddings',
- model_version: '2026-09-30',
- manifest_version: 'manifest',
+ status: 'scored',
+ model: 'report_embeddings@2026-09-30',
scored_at: '2026-09-30T12:00:00Z',
- scores: { pr_merged: 0.4 },
- lifts: {},
- readable_heads: ['pr_merged'],
- stale: false,
+ predictions: [{ outcome: 'fix', probability: 0.4, lift: null, readable: true }],
},
}),
makeReport({ id: 'b', priority: null, actionability: null, source_products: ['error_tracking'] }),
@@ -144,7 +139,7 @@ describe('inboxAnalytics', () => {
sortField: 'ranking_pr_merged',
sortDirection: 'desc',
createdWindow: '7d',
- rankerId: 'fixed',
+ rankerId: 'fix',
})
const props = lastCapture(INBOX_EVENTS.REPORTS_IMPRESSED)
expect(props).toMatchObject({
@@ -155,7 +150,7 @@ describe('inboxAnalytics', () => {
sort_field: 'ranking_pr_merged',
sort_direction: 'desc',
created_window: '7d',
- ranker_id: 'fixed',
+ ranker_id: 'fix',
})
expect(props?.impressions).toEqual([
expect.objectContaining({
@@ -178,7 +173,7 @@ describe('inboxAnalytics', () => {
})
it('emits a single-report action with the report context and the ranker of the list in view', () => {
- setInboxServedRanker('fixed')
+ setInboxServedRanker('fix')
captureInboxReportAction({
report: makeReport(),
actionType: 'create_pr',
@@ -190,12 +185,12 @@ describe('inboxAnalytics', () => {
surface: 'detail_pane',
is_bulk: false,
bulk_size: 1,
- ranker_id: 'fixed',
+ ranker_id: 'fix',
})
})
it('never attributes a Today action to the Inbox ranker', () => {
- setInboxServedRanker('fixed')
+ setInboxServedRanker('fix')
captureInboxReportAction({ report: makeReport(), actionType: 'discuss', surface: 'today' })
expect(lastCapture(INBOX_EVENTS.REPORT_ACTION)?.ranker_id).toBeNull()
})
diff --git a/products/signals/frontend/inbox/inboxAnalytics.ts b/products/signals/frontend/inbox/inboxAnalytics.ts
index f8939bc46489..7b56affcb22b 100644
--- a/products/signals/frontend/inbox/inboxAnalytics.ts
+++ b/products/signals/frontend/inbox/inboxAnalytics.ts
@@ -496,7 +496,7 @@ export function captureInboxReportsImpressed(params: {
signal_count: report.signal_count,
total_weight: report.total_weight,
is_suggested_reviewer: report.is_suggested_reviewer,
- ...(rankingSort ? { ranking_served_key: report.ranking?.served_key ?? null } : {}),
+ ...(rankingSort ? { ranking_served_key: report.ranking?.model ?? null } : {}),
})),
})
}
diff --git a/products/signals/frontend/inbox/logics/inboxFiltersLogic.test.ts b/products/signals/frontend/inbox/logics/inboxFiltersLogic.test.ts
index 2675620b4975..3b6df8648a5d 100644
--- a/products/signals/frontend/inbox/logics/inboxFiltersLogic.test.ts
+++ b/products/signals/frontend/inbox/logics/inboxFiltersLogic.test.ts
@@ -58,7 +58,7 @@ describe('inboxFiltersLogic', () => {
})
it('leads with the ranking field for a model sort', () => {
- expect(buildSignalReportListOrdering('ranking_fixed', 'desc')).toBe('-ranking_fixed,status,-updated_at')
+ expect(buildSignalReportListOrdering('ranking_fix', 'desc')).toBe('-ranking_fix,status,-updated_at')
})
// The server ignores the `sort` preset whenever `ordering` is present.
@@ -154,6 +154,13 @@ describe('inboxFiltersLogic', () => {
'asc',
],
['keeps an explicit Priority first from a shared link', 'priority:asc', false, 'priority', 'asc'],
+ [
+ 'moves a link with the old Fix sort name to the current one',
+ 'ranking_fixed:desc',
+ true,
+ 'ranking_fix',
+ 'desc',
+ ],
] as const)('%s', (_name, sort, modelSortAvailable, sortField, sortDirection) => {
expect(parseFilterSearchParams({ sort }, { modelSortAvailable })).toEqual({
...DEFAULT_STATE,
diff --git a/products/signals/frontend/inbox/logics/inboxFiltersLogic.ts b/products/signals/frontend/inbox/logics/inboxFiltersLogic.ts
index eb0a8ee1ee76..467e9862e54f 100644
--- a/products/signals/frontend/inbox/logics/inboxFiltersLogic.ts
+++ b/products/signals/frontend/inbox/logics/inboxFiltersLogic.ts
@@ -40,7 +40,7 @@ export interface InboxReviewerOption {
email: string
}
-export type InboxRankingSortField = 'ranking_open' | 'ranking_action' | 'ranking_fixed'
+export type InboxRankingSortField = 'ranking_open' | 'ranking_action' | 'ranking_fix'
export type InboxSortField = 'recommended' | 'priority' | 'created_at' | 'updated_at' | InboxRankingSortField
export type InboxSortDirection = 'asc' | 'desc'
/** Preset for the created-in window filter. Null means any time. */
@@ -87,6 +87,13 @@ const VALID_SORT_KEYS = new Set(INBOX_SORT_OPTIONS.map((o) => `${o.field}:${o.di
const VALID_MODEL_SORT_KEYS = new Set(INBOX_MODEL_SORT_OPTIONS.map((o) => `${o.field}:${o.direction}`))
const VALID_CREATED_WINDOWS = new Set<string>(INBOX_CREATED_WINDOW_OPTIONS.map((o) => o.value))
+// Old sort field names that URLs and local storage can still hold, so a stored sort does not fall back to the default.
+const RENAMED_SORT_FIELDS: Record<string, InboxSortField> = { ranking_fixed: 'ranking_fix' }
+
+function currentSortField(field: string): InboxSortField {
+ return RENAMED_SORT_FIELDS[field] ?? (field as InboxSortField)
+}
+
function isOfferedSortKey(key: string, modelSortAvailable: boolean): boolean {
return VALID_SORT_KEYS.has(key) || (modelSortAvailable && VALID_MODEL_SORT_KEYS.has(key))
}
@@ -163,10 +170,13 @@ export function parseFilterSearchParams(
): InboxFilterState {
let sortField = DEFAULT_SORT_FIELD
let sortDirection = DEFAULT_SORT_DIRECTION
- if (typeof searchParams.sort === 'string' && isOfferedSortKey(searchParams.sort, modelSortAvailable)) {
- const [field, direction] = searchParams.sort.split(':')
- sortField = field as InboxSortField
- sortDirection = direction as InboxSortDirection
+ if (typeof searchParams.sort === 'string') {
+ const [rawField, direction] = searchParams.sort.split(':')
+ const field = currentSortField(rawField)
+ if (isOfferedSortKey(`${field}:${direction}`, modelSortAvailable)) {
+ sortField = field
+ sortDirection = direction as InboxSortDirection
+ }
}
return {
scope: parseScopeParam(searchParams.scope),
@@ -762,7 +772,9 @@ export const inboxFiltersLogic = kea<inboxFiltersLogicType>([
sortDirection: InboxSortDirection,
modelSortAvailable: boolean
): InboxSortField =>
- isOfferedSort(sortField, sortDirection, modelSortAvailable) ? sortField : DEFAULT_SORT_FIELD,
+ isOfferedSort(currentSortField(sortField), sortDirection, modelSortAvailable)
+ ? currentSortField(sortField)
+ : DEFAULT_SORT_FIELD,
],
activeSortDirection: [
(s) => [s.sortField, s.sortDirection, s.modelSortAvailable],
@@ -771,7 +783,9 @@ export const inboxFiltersLogic = kea<inboxFiltersLogicType>([
sortDirection: InboxSortDirection,
modelSortAvailable: boolean
): InboxSortDirection =>
- isOfferedSort(sortField, sortDirection, modelSortAvailable) ? sortDirection : DEFAULT_SORT_DIRECTION,
+ isOfferedSort(currentSortField(sortField), sortDirection, modelSortAvailable)
+ ? sortDirection
+ : DEFAULT_SORT_DIRECTION,
],
timeWindowAvailable: [
(s) => [s.featureFlags],
diff --git a/products/signals/frontend/inbox/types.ts b/products/signals/frontend/inbox/types.ts
index 515e9a0712d2..36339b568700 100644
--- a/products/signals/frontend/inbox/types.ts
+++ b/products/signals/frontend/inbox/types.ts
@@ -152,7 +152,7 @@ export interface SignalReport extends Pick<SignalReportApi, 'monitoring_started_
billing_exempt_reason?: string | null
/** Backend-owned refund eligibility: why a refund would be rejected right now, null when it would be accepted. */
refund_ineligibility_reason?: string | null
- /** The served ranking model's score. Null for users outside staff and the model sort flag, and for unscored reports. Raw scores and model metadata are staff only. */
+ /** The served ranking model's predictions. Its status says whether the report is scored, stale or unscored, or that the user cannot see the ranking model. */
ranking?: ReportRankingApi | null
/** Coarse sort key from the list response's ranker, for the Recommended sort only. Lower keys rank first. */
inbox_merge_key?: number | null
diff --git a/products/signals/frontend/inbox/utils/reportOrdering.test.ts b/products/signals/frontend/inbox/utils/reportOrdering.test.ts
index f6165ce0c87a..57de8932c85c 100644
--- a/products/signals/frontend/inbox/utils/reportOrdering.test.ts
+++ b/products/signals/frontend/inbox/utils/reportOrdering.test.ts
@@ -66,25 +66,20 @@ describe('compareSignalReports', () => {
rows.sort(compareSignalReports('created_at', 'desc'))
expect(rows.map((r) => r.id)).toEqual(['fractional', 'whole-second'])
})
- // Non-staff users get lifts without probabilities, and the merged list must keep the server's model order.
- it('orders a model sort by the lift when the report has no probability', () => {
- const ranking = {
- served_key: '',
- model_name: '',
- model_version: '',
- manifest_version: '',
+ it('orders a model sort by the probability and puts unscored and stale reports last', () => {
+ const ranking = (probability: number, status: 'scored' | 'stale' = 'scored'): SignalReport['ranking'] => ({
+ status,
+ model: 'report_embeddings@2026-06-01',
scored_at: '2026-06-01T00:00:00Z',
- scores: {},
- readable_heads: ['fixed'],
- stale: false,
- }
+ predictions: [{ outcome: 'fix', probability, lift: null, readable: true }],
+ })
const rows = [
report('unscored', { ranking: null }),
- report('low', { ranking: { ...ranking, lifts: { fixed: 0.5 } } }),
- report('stale-high', { ranking: { ...ranking, lifts: { fixed: 9 }, stale: true } }),
- report('high', { ranking: { ...ranking, lifts: { fixed: 3 } } }),
+ report('low', { ranking: ranking(0.1) }),
+ report('stale-high', { ranking: ranking(0.9, 'stale') }),
+ report('high', { ranking: ranking(0.5) }),
]
- rows.sort(compareSignalReports('ranking_fixed', 'desc'))
+ rows.sort(compareSignalReports('ranking_fix', 'desc'))
expect(rows.map((r) => r.id)).toEqual(['high', 'low', 'unscored', 'stale-high'])
})
})
diff --git a/products/signals/frontend/inbox/utils/reportOrdering.ts b/products/signals/frontend/inbox/utils/reportOrdering.ts
index fe402caa6c98..4e6f6a5319fb 100644
--- a/products/signals/frontend/inbox/utils/reportOrdering.ts
+++ b/products/signals/frontend/inbox/utils/reportOrdering.ts
@@ -1,3 +1,5 @@
+import type { ReportRankingPredictionApi } from 'products/signals/frontend/generated/api.schemas'
+
import { isRankingSortField, rankingScoreForField } from '../filterOptions'
import type { InboxRankingSortField, InboxSortDirection, InboxSortField } from '../logics/inboxFiltersLogic'
import { SignalReport, SignalReportStatus } from '../types'
@@ -48,18 +50,21 @@ function timestampKey(value: string): string {
return value.includes('.') ? value : value.replace(/(Z|[+-]\d\d:?\d\d)$/, '.000000$1')
}
-/**
- * The served probability a model sort orders by, or null when the report has no score for that head or the score is stale.
- * Non-staff users get no probabilities, so their rows order by the lift. The lift is the probability over the head's
- * base rate, so it gives the same order.
- */
+/** The served model's prediction for one outcome, or undefined when the report has none. */
+export function rankingPrediction(
+ ranking: SignalReport['ranking'],
+ outcome: string
+): ReportRankingPredictionApi | undefined {
+ return ranking?.predictions.find((prediction) => prediction.outcome === outcome)
+}
+
+/** The served probability a model sort orders by, or null when the report has no score for that outcome or the score is stale. */
export function rankingSortScore(report: SignalReport, field: InboxRankingSortField): number | null {
- if (report.ranking?.stale) {
+ if (report.ranking?.status !== 'scored') {
return null
}
- const head = rankingScoreForField(field).head
- const score = report.ranking?.scores[head] ?? report.ranking?.lifts?.[head]
- return typeof score === 'number' ? score : null
+ const probability = rankingPrediction(report.ranking, rankingScoreForField(field).outcome)?.probability
+ return typeof probability === 'number' ? probability : null
}
/**
diff --git a/products/signals/mcp/tools.yaml b/products/signals/mcp/tools.yaml
index 5376ed6b212f..9bfe55952748 100644
--- a/products/signals/mcp/tools.yaml
+++ b/products/signals/mcp/tools.yaml
@@ -31,6 +31,23 @@ tools:
one version with its full profile. A 404 means activity summaries are not available for this project, or
that the requested version does not exist.
feature_flag: activity-summary
+ inbox-ranking-outcomes-list:
+ operation: signals_reports_ranking_outcomes_list
+ enabled: true
+ scopes:
+ - task:read
+ annotations:
+ readOnly: true
+ destructive: false
+ idempotent: true
+ title: List inbox ranking outcomes
+ description: >
+ List the outcomes the inbox ranking model predicts for each report (`open`, `action`, `fix`), with what each
+ one means, its horizon in days, its cohort, its base rate, and whether training could read it. The values
+ come from the model that scored this project's reports, so read this tool for the current meaning of the
+ `ranking.predictions` field in inbox-reports-list rather than relying on a fixed description. A lift is the
+ probability divided by the base rate. A prediction is a forecast, never the report's state. `status:
+ not_available` means you cannot see the ranking model.
inbox-report-actions-list:
operation: signals_reports_actions_list
enabled: true
@@ -416,6 +433,11 @@ tools:
`source_id`); add `include_all_statuses=true` to include dismissed ones. Each row includes typed impact
metric metadata and any accessible cached snapshot, but omits live query definitions and comparisons.
Queryless legacy or malformed metrics are always redacted. Listing reports does not execute metric queries.
+ Each row's `ranking.predictions` carries the inbox ranking model's predicted probability and lift per
+ outcome (`open`, `action`, `fix`). Lift 1.0 is an average report. A prediction is a forecast, not the
+ report's state, and `status: unscored` is not a low score. Order with `ordering=-ranking_<outcome>`, filter
+ with `ranking_min_lift` and `ranking_min_probability` (for example `fix:2`), and read
+ inbox-ranking-outcomes-list for what each outcome means.
agent_note: >
You may inspect reports without claiming them. A claim indicates active work that should not be duplicated.
Before claiming a report, read the report and its work log. If you decide to begin working to fix the issues
@@ -455,6 +477,9 @@ tools:
- tracker_issue_error
- work_state
- assignee
+ - ranking.status
+ - ranking.scored_at
+ - ranking.predictions
- created_at
- updated_at
inbox-reports-merge:
@@ -507,7 +532,8 @@ tools:
are always redacted. The report's full work log is readable via inbox-report-artefacts-list; read it before
acting. The raw underlying signals require a HogQL query (see the 'signals' skill). Claim the report before
doing work. A report may be claimed in potential, candidate, in_progress, pending_input, ready, or failed
- state.
+ state. Its `ranking` field has the ranking model's predictions and the key of the model that made them, as
+ described in inbox-reports-list.
agent_note: >
You may inspect reports without claiming them. A claim indicates active work that should not be duplicated.
Before claiming a report, read the report and its work log. If you decide to begin working to fix the issues
diff --git a/products/signals/skills/inbox-exploration/SKILL.md b/products/signals/skills/inbox-exploration/SKILL.md
index b05f797b463a..c0743f20d65f 100644
--- a/products/signals/skills/inbox-exploration/SKILL.md
+++ b/products/signals/skills/inbox-exploration/SKILL.md
@@ -69,6 +69,7 @@ _underlying detail_ — pair them when the user wants to dig in.
| `inbox-reports-bulk-set-state` | Same transition for 1–100 reports in one call (per-id result) |
| `inbox-report-actions-list` | The report's agent run actions (Get it merged, suggested prompts) and whether each can start now (read-only) |
| `inbox-report-actions-run` | Start one of those actions as a background agent run linked to the report |
+| `inbox-ranking-outcomes-list` | What each ranking model outcome predicts, with its horizon, cohort and base rate (read-only) |
| `inbox-source-configs-list` | Configured signal sources (which products feed the inbox) |
| `inbox-source-configs-retrieve` | Full record for a single source config |
| `inbox-source-configs-partial-update` | Toggle a source's `enabled` flag (or adjust its `config`) |
@@ -595,6 +596,54 @@ inbox-reports-list
If the keyword search returns nothing meaningful, hand off to the `signals` skill — semantic
search over signal text via `embedText()` will catch reports the keyword filter missed.
+## Model predictions as building blocks
+
+The inbox ranking model predicts outcomes for each report. Each row of `inbox-reports-list` and
+`inbox-reports-retrieve` carries them in `ranking`:
+
+```json
+"ranking": {
+ "status": "scored",
+ "scored_at": "2026-10-24T05:12:00Z",
+ "predictions": [
+ { "outcome": "open", "probability": 0.21, "lift": 1.5, "readable": true },
+ { "outcome": "action", "probability": 0.09, "lift": 1.4, "readable": true },
+ { "outcome": "fix", "probability": 0.3, "lift": 2.7, "readable": true }
+ ]
+}
+```
+
+- `open`: a person opens the report. `action`: a person or their agent acts on it. `fix`:
+ someone fixes the problem it describes. Call `inbox-ranking-outcomes-list` for the current
+ meaning, horizon and base rate of each outcome, because they follow the served model.
+- A prediction says what may happen, never what did. A high `fix` probability does not mean the
+ report is fixed: read `status` and the PR fields for that.
+- A lift is the probability divided by the outcome's base rate. 1.0 is an average report, 2.7 is
+ 2.7x as likely. Compare lifts across outcomes. Raw probabilities do not compare, because each
+ outcome has its own base rate.
+- `status` is `scored`, `stale` (the title or summary changed after scoring, so the predictions
+ describe the old text), `unscored` (no score yet, which is not a low score), or `not_available`
+ (the user cannot see the ranking model, so ordering and filtering by it return 400).
+- `inbox-reports-retrieve` also names the `model` that made the predictions. Predictions from
+ different models do not compare.
+
+Recipes:
+
+- Likely to get a fix: `view=actionable` with `ordering=-ranking_fix`.
+- Hidden gems: `priority=P3,P4` with `ranking_min_lift=fix:2`.
+- Probably noise: reports with a low lift on all three outcomes are candidates to dedupe or
+ dismiss. Read them before you act: a low lift is a hint, not a verdict.
+- Add `search` or `source_product` to any recipe to narrow it to a topic.
+
+`ranking_min_lift` and `ranking_min_probability` take comma-separated `<outcome>:<value>` pairs,
+and a report must meet every pair. Unscored and stale reports never match.
+
+Caveats:
+
+- The predictions are model estimates, and the top end over-predicts. Use them to rank and
+ filter, not as literal odds.
+- Most `fix` positives are also `action` positives, so do not add the two together.
+
## Workflow: review configured sources
When the user asks "which signal sources are set up?" or "is <product> hooked up?":
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index 5fed4165c8e1..19968e12818b 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -7608,6 +7608,21 @@
"readOnlyHint": true
}
},
+ "inbox-ranking-outcomes-list": {
+ "description": "List the outcomes the inbox ranking model predicts for each report (`open`, `action`, `fix`), with what each one means, its horizon in days, its cohort, its base rate, and whether training could read it. The values come from the model that scored this project's reports, so read this tool for the current meaning of the `ranking.predictions` field in inbox-reports-list rather than relying on a fixed description. A lift is the probability divided by the base rate. A prediction is a forecast, never the report's state. `status: not_available` means you cannot see the ranking model.",
+ "category": "Signals",
+ "feature": "signals",
+ "summary": "List inbox ranking outcomes",
+ "title": "List inbox ranking outcomes",
+ "required_scopes": ["task:read"],
+ "api_operations": ["GET /api/projects/{project_id}/signals/reports/ranking_outcomes/"],
+ "annotations": {
+ "destructiveHint": false,
+ "idempotentHint": true,
+ "openWorldHint": true,
+ "readOnlyHint": true
+ }
+ },
"inbox-report-actions-list": {
"description": "List the actions on a signal report that start an agent run, the same ones the inbox shows as buttons: `get_it_merged` (fix CI on the report's open, approved pull request and merge it), `make_it_green` (fix failing CI on the report's open pull request and stop without merging it), and one `suggested_prompt` entry per prompt the report suggests. Every action comes back, available or not. `available` says whether a run can start now, and `disabled_reason` says why not. `text` is the message the run gets as its first request. Pass `action`, and `suggested_prompt_index` for a suggested prompt, to inbox-report-actions-run.",
"category": "Signals",
@@ -7855,7 +7870,7 @@
}
},
"inbox-reports-list": {
- "description": "List signal reports for the current project. A signal report is a cluster of related observations (signals) that PostHog has aggregated into a single issue or trend. Reports surface in the Inbox. Supports filtering by status (potential, candidate, in_progress, pending_input, ready, resolved, failed, suppressed), free-text search across title and summary, source_product (e.g. error_tracking, session_replay), suggested_reviewers, `unclaimed`, and `assignee=me`. Claimed reports remain actionable and visible; use `unclaimed=true` to pick work that has no owner and no draft, open, or unknown PR. Some statuses are hidden by default (currently suppressed); pass `include_all_statuses=true` to list reports in every status. Do this when deduplicating against the full inbox state, and read each row's `status` (plus `suppression_source`, `dismissal_reason`, and `dismissal_note` on suppressed rows) before acting. A suppressed row's `suppression_source` says whether someone dismissed it (`dismissed`) or a judge held it back before anyone saw it (`safety_judge`, `not_actionable`, `system`). For picking up work, filter to `ready` — earlier statuses are still moving through the pipeline, and `pending_input` reports are waiting on a human. Each report's full work log, including signal findings, judgments, and log entries, is readable via inbox-report-artefacts-list; read it before acting on a report. Results are paginated and ordered by '-is_suggested_reviewer,status,-updated_at' by default. To answer \"what should I look at next\", pass `view=actionable` with `use_priority_preference=true`, `sort=priority`, and `limit=10`: that returns ready, actionable reports that have no implementation PR yet, at or above the requesting user's personal PR-generation threshold, falling back to the project threshold. `use_priority_preference` selects which priorities to include, and does not order them, so pass `sort=priority` to put P0 first; the default ordering can put a recently updated P4 above an older P0. Without `limit` a page holds up to 100 reports, each with its summary. To answer \"what needs me\", pass `view=for_you` with `sort=priority`: that returns the open, actionable reports that wait for the requesting user's input, that they claimed, or that name them as a suggested reviewer, and `count` gives how many there are in total. The other views are `needs_input` (waiting on a human), `needs_decision` (failed reports, plus actionable ready or pending-input reports without an implementation PR), `monitoring` (an implementation PR is open), `resolved`, `dismissed` (suppressed by a person, an agent, a merge, or a closed PR), `held_back` (suppressed by a judge, with nobody's review yet), `not_actionable`, and `all`. Narrow by reviewer with `scope=for_me`, or `scope=teammate` plus `teammate_uuid`; the default `scope=entire_project` searches every report. `priority`, `source_product`, and `scout` each take a comma-separated list. To find the self-driving reports for one source record — e.g. what the inbox already investigated for a specific support ticket — pass that record's id as `source_id` together with exactly one `source_product` (for a support ticket: `source_product=conversations` and the ticket's UUID as `source_id`); add `include_all_statuses=true` to include dismissed ones. Each row includes typed impact metric metadata and any accessible cached snapshot, but omits live query definitions and comparisons. Queryless legacy or malformed metrics are always redacted. Listing reports does not execute metric queries.",
+ "description": "List signal reports for the current project. A signal report is a cluster of related observations (signals) that PostHog has aggregated into a single issue or trend. Reports surface in the Inbox. Supports filtering by status (potential, candidate, in_progress, pending_input, ready, resolved, failed, suppressed), free-text search across title and summary, source_product (e.g. error_tracking, session_replay), suggested_reviewers, `unclaimed`, and `assignee=me`. Claimed reports remain actionable and visible; use `unclaimed=true` to pick work that has no owner and no draft, open, or unknown PR. Some statuses are hidden by default (currently suppressed); pass `include_all_statuses=true` to list reports in every status. Do this when deduplicating against the full inbox state, and read each row's `status` (plus `suppression_source`, `dismissal_reason`, and `dismissal_note` on suppressed rows) before acting. A suppressed row's `suppression_source` says whether someone dismissed it (`dismissed`) or a judge held it back before anyone saw it (`safety_judge`, `not_actionable`, `system`). For picking up work, filter to `ready` — earlier statuses are still moving through the pipeline, and `pending_input` reports are waiting on a human. Each report's full work log, including signal findings, judgments, and log entries, is readable via inbox-report-artefacts-list; read it before acting on a report. Results are paginated and ordered by '-is_suggested_reviewer,status,-updated_at' by default. To answer \"what should I look at next\", pass `view=actionable` with `use_priority_preference=true`, `sort=priority`, and `limit=10`: that returns ready, actionable reports that have no implementation PR yet, at or above the requesting user's personal PR-generation threshold, falling back to the project threshold. `use_priority_preference` selects which priorities to include, and does not order them, so pass `sort=priority` to put P0 first; the default ordering can put a recently updated P4 above an older P0. Without `limit` a page holds up to 100 reports, each with its summary. To answer \"what needs me\", pass `view=for_you` with `sort=priority`: that returns the open, actionable reports that wait for the requesting user's input, that they claimed, or that name them as a suggested reviewer, and `count` gives how many there are in total. The other views are `needs_input` (waiting on a human), `needs_decision` (failed reports, plus actionable ready or pending-input reports without an implementation PR), `monitoring` (an implementation PR is open), `resolved`, `dismissed` (suppressed by a person, an agent, a merge, or a closed PR), `held_back` (suppressed by a judge, with nobody's review yet), `not_actionable`, and `all`. Narrow by reviewer with `scope=for_me`, or `scope=teammate` plus `teammate_uuid`; the default `scope=entire_project` searches every report. `priority`, `source_product`, and `scout` each take a comma-separated list. To find the self-driving reports for one source record — e.g. what the inbox already investigated for a specific support ticket — pass that record's id as `source_id` together with exactly one `source_product` (for a support ticket: `source_product=conversations` and the ticket's UUID as `source_id`); add `include_all_statuses=true` to include dismissed ones. Each row includes typed impact metric metadata and any accessible cached snapshot, but omits live query definitions and comparisons. Queryless legacy or malformed metrics are always redacted. Listing reports does not execute metric queries. Each row's `ranking.predictions` carries the inbox ranking model's predicted probability and lift per outcome (`open`, `action`, `fix`). Lift 1.0 is an average report. A prediction is a forecast, not the report's state, and `status: unscored` is not a low score. Order with `ordering=-ranking_<outcome>`, filter with `ranking_min_lift` and `ranking_min_probability` (for example `fix:2`), and read inbox-ranking-outcomes-list for what each outcome means.",
"category": "Signals",
"feature": "signals",
"summary": "List signal reports",
@@ -7885,7 +7900,7 @@
}
},
"inbox-reports-retrieve": {
- "description": "Get a single signal report by ID. Returns the full report, including work_state, current assignee, and the attached pull request and its latest known state, plus typed impact metrics. Every metric includes a bounded live InsightVizNode/TrendsQuery built only from EventsNode or ActionsNode sources and may include an optional cached snapshot fallback. This call returns that stored content without executing the query. Consumers derive a BoldNumber whole-window headline and an ActionsBar longitudinal response, capped at 1,000 estimated points, through the normal analytics query path and cache. Queryless legacy or malformed metrics are always redacted. The report's full work log is readable via inbox-report-artefacts-list; read it before acting. The raw underlying signals require a HogQL query (see the 'signals' skill). Claim the report before doing work. A report may be claimed in potential, candidate, in_progress, pending_input, ready, or failed state.",
+ "description": "Get a single signal report by ID. Returns the full report, including work_state, current assignee, and the attached pull request and its latest known state, plus typed impact metrics. Every metric includes a bounded live InsightVizNode/TrendsQuery built only from EventsNode or ActionsNode sources and may include an optional cached snapshot fallback. This call returns that stored content without executing the query. Consumers derive a BoldNumber whole-window headline and an ActionsBar longitudinal response, capped at 1,000 estimated points, through the normal analytics query path and cache. Queryless legacy or malformed metrics are always redacted. The report's full work log is readable via inbox-report-artefacts-list; read it before acting. The raw underlying signals require a HogQL query (see the 'signals' skill). Claim the report before doing work. A report may be claimed in potential, candidate, in_progress, pending_input, ready, or failed state. Its `ranking` field has the ranking model's predictions and the key of the model that made them, as described in inbox-reports-list.",
"category": "Signals",
"feature": "signals",
"summary": "Get signal report",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index 36e158624b80..9c97c674e247 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -7810,6 +7810,21 @@
"readOnlyHint": true
}
},
+ "inbox-ranking-outcomes-list": {
+ "description": "List the outcomes the inbox ranking model predicts for each report (`open`, `action`, `fix`), with what each one means, its horizon in days, its cohort, its base rate, and whether training could read it. The values come from the model that scored this project's reports, so read this tool for the current meaning of the `ranking.predictions` field in inbox-reports-list rather than relying on a fixed description. A lift is the probability divided by the base rate. A prediction is a forecast, never the report's state. `status: not_available` means you cannot see the ranking model.",
+ "category": "Signals",
+ "feature": "signals",
+ "summary": "List inbox ranking outcomes",
+ "title": "List inbox ranking outcomes",
+ "required_scopes": ["task:read"],
+ "api_operations": ["GET /api/projects/{project_id}/signals/reports/ranking_outcomes/"],
+ "annotations": {
+ "destructiveHint": false,
+ "idempotentHint": true,
+ "openWorldHint": true,
+ "readOnlyHint": true
+ }
+ },
"inbox-report-actions-list": {
"description": "List the actions on a signal report that start an agent run, the same ones the inbox shows as buttons: `get_it_merged` (fix CI on the report's open, approved pull request and merge it), `make_it_green` (fix failing CI on the report's open pull request and stop without merging it), and one `suggested_prompt` entry per prompt the report suggests. Every action comes back, available or not. `available` says whether a run can start now, and `disabled_reason` says why not. `text` is the message the run gets as its first request. Pass `action`, and `suggested_prompt_index` for a suggested prompt, to inbox-report-actions-run.",
"category": "Signals",
@@ -8057,7 +8072,7 @@
}
},
"inbox-reports-list": {
- "description": "List signal reports for the current project. A signal report is a cluster of related observations (signals) that PostHog has aggregated into a single issue or trend. Reports surface in the Inbox. Supports filtering by status (potential, candidate, in_progress, pending_input, ready, resolved, failed, suppressed), free-text search across title and summary, source_product (e.g. error_tracking, session_replay), suggested_reviewers, `unclaimed`, and `assignee=me`. Claimed reports remain actionable and visible; use `unclaimed=true` to pick work that has no owner and no draft, open, or unknown PR. Some statuses are hidden by default (currently suppressed); pass `include_all_statuses=true` to list reports in every status. Do this when deduplicating against the full inbox state, and read each row's `status` (plus `suppression_source`, `dismissal_reason`, and `dismissal_note` on suppressed rows) before acting. A suppressed row's `suppression_source` says whether someone dismissed it (`dismissed`) or a judge held it back before anyone saw it (`safety_judge`, `not_actionable`, `system`). For picking up work, filter to `ready` — earlier statuses are still moving through the pipeline, and `pending_input` reports are waiting on a human. Each report's full work log, including signal findings, judgments, and log entries, is readable via inbox-report-artefacts-list; read it before acting on a report. Results are paginated and ordered by '-is_suggested_reviewer,status,-updated_at' by default. To answer \"what should I look at next\", pass `view=actionable` with `use_priority_preference=true`, `sort=priority`, and `limit=10`: that returns ready, actionable reports that have no implementation PR yet, at or above the requesting user's personal PR-generation threshold, falling back to the project threshold. `use_priority_preference` selects which priorities to include, and does not order them, so pass `sort=priority` to put P0 first; the default ordering can put a recently updated P4 above an older P0. Without `limit` a page holds up to 100 reports, each with its summary. To answer \"what needs me\", pass `view=for_you` with `sort=priority`: that returns the open, actionable reports that wait for the requesting user's input, that they claimed, or that name them as a suggested reviewer, and `count` gives how many there are in total. The other views are `needs_input` (waiting on a human), `needs_decision` (failed reports, plus actionable ready or pending-input reports without an implementation PR), `monitoring` (an implementation PR is open), `resolved`, `dismissed` (suppressed by a person, an agent, a merge, or a closed PR), `held_back` (suppressed by a judge, with nobody's review yet), `not_actionable`, and `all`. Narrow by reviewer with `scope=for_me`, or `scope=teammate` plus `teammate_uuid`; the default `scope=entire_project` searches every report. `priority`, `source_product`, and `scout` each take a comma-separated list. To find the self-driving reports for one source record — e.g. what the inbox already investigated for a specific support ticket — pass that record's id as `source_id` together with exactly one `source_product` (for a support ticket: `source_product=conversations` and the ticket's UUID as `source_id`); add `include_all_statuses=true` to include dismissed ones. Each row includes typed impact metric metadata and any accessible cached snapshot, but omits live query definitions and comparisons. Queryless legacy or malformed metrics are always redacted. Listing reports does not execute metric queries.",
+ "description": "List signal reports for the current project. A signal report is a cluster of related observations (signals) that PostHog has aggregated into a single issue or trend. Reports surface in the Inbox. Supports filtering by status (potential, candidate, in_progress, pending_input, ready, resolved, failed, suppressed), free-text search across title and summary, source_product (e.g. error_tracking, session_replay), suggested_reviewers, `unclaimed`, and `assignee=me`. Claimed reports remain actionable and visible; use `unclaimed=true` to pick work that has no owner and no draft, open, or unknown PR. Some statuses are hidden by default (currently suppressed); pass `include_all_statuses=true` to list reports in every status. Do this when deduplicating against the full inbox state, and read each row's `status` (plus `suppression_source`, `dismissal_reason`, and `dismissal_note` on suppressed rows) before acting. A suppressed row's `suppression_source` says whether someone dismissed it (`dismissed`) or a judge held it back before anyone saw it (`safety_judge`, `not_actionable`, `system`). For picking up work, filter to `ready` — earlier statuses are still moving through the pipeline, and `pending_input` reports are waiting on a human. Each report's full work log, including signal findings, judgments, and log entries, is readable via inbox-report-artefacts-list; read it before acting on a report. Results are paginated and ordered by '-is_suggested_reviewer,status,-updated_at' by default. To answer \"what should I look at next\", pass `view=actionable` with `use_priority_preference=true`, `sort=priority`, and `limit=10`: that returns ready, actionable reports that have no implementation PR yet, at or above the requesting user's personal PR-generation threshold, falling back to the project threshold. `use_priority_preference` selects which priorities to include, and does not order them, so pass `sort=priority` to put P0 first; the default ordering can put a recently updated P4 above an older P0. Without `limit` a page holds up to 100 reports, each with its summary. To answer \"what needs me\", pass `view=for_you` with `sort=priority`: that returns the open, actionable reports that wait for the requesting user's input, that they claimed, or that name them as a suggested reviewer, and `count` gives how many there are in total. The other views are `needs_input` (waiting on a human), `needs_decision` (failed reports, plus actionable ready or pending-input reports without an implementation PR), `monitoring` (an implementation PR is open), `resolved`, `dismissed` (suppressed by a person, an agent, a merge, or a closed PR), `held_back` (suppressed by a judge, with nobody's review yet), `not_actionable`, and `all`. Narrow by reviewer with `scope=for_me`, or `scope=teammate` plus `teammate_uuid`; the default `scope=entire_project` searches every report. `priority`, `source_product`, and `scout` each take a comma-separated list. To find the self-driving reports for one source record — e.g. what the inbox already investigated for a specific support ticket — pass that record's id as `source_id` together with exactly one `source_product` (for a support ticket: `source_product=conversations` and the ticket's UUID as `source_id`); add `include_all_statuses=true` to include dismissed ones. Each row includes typed impact metric metadata and any accessible cached snapshot, but omits live query definitions and comparisons. Queryless legacy or malformed metrics are always redacted. Listing reports does not execute metric queries. Each row's `ranking.predictions` carries the inbox ranking model's predicted probability and lift per outcome (`open`, `action`, `fix`). Lift 1.0 is an average report. A prediction is a forecast, not the report's state, and `status: unscored` is not a low score. Order with `ordering=-ranking_<outcome>`, filter with `ranking_min_lift` and `ranking_min_probability` (for example `fix:2`), and read inbox-ranking-outcomes-list for what each outcome means.",
"category": "Signals",
"feature": "signals",
"summary": "List signal reports",
@@ -8087,7 +8102,7 @@
}
},
"inbox-reports-retrieve": {
- "description": "Get a single signal report by ID. Returns the full report, including work_state, current assignee, and the attached pull request and its latest known state, plus typed impact metrics. Every metric includes a bounded live InsightVizNode/TrendsQuery built only from EventsNode or ActionsNode sources and may include an optional cached snapshot fallback. This call returns that stored content without executing the query. Consumers derive a BoldNumber whole-window headline and an ActionsBar longitudinal response, capped at 1,000 estimated points, through the normal analytics query path and cache. Queryless legacy or malformed metrics are always redacted. The report's full work log is readable via inbox-report-artefacts-list; read it before acting. The raw underlying signals require a HogQL query (see the 'signals' skill). Claim the report before doing work. A report may be claimed in potential, candidate, in_progress, pending_input, ready, or failed state.",
+ "description": "Get a single signal report by ID. Returns the full report, including work_state, current assignee, and the attached pull request and its latest known state, plus typed impact metrics. Every metric includes a bounded live InsightVizNode/TrendsQuery built only from EventsNode or ActionsNode sources and may include an optional cached snapshot fallback. This call returns that stored content without executing the query. Consumers derive a BoldNumber whole-window headline and an ActionsBar longitudinal response, capped at 1,000 estimated points, through the normal analytics query path and cache. Queryless legacy or malformed metrics are always redacted. The report's full work log is readable via inbox-report-artefacts-list; read it before acting. The raw underlying signals require a HogQL query (see the 'signals' skill). Claim the report before doing work. A report may be claimed in potential, candidate, in_progress, pending_input, ready, or failed state. Its `ranking` field has the ranking model's predictions and the key of the model that made them, as described in inbox-reports-list.",
"category": "Signals",
"feature": "signals",
"summary": "Get signal report",
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 09eeb38e2d32..248b4baea30f 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -78159,34 +78159,73 @@ export namespace Schemas {
} as const;
/**
- * Outcome head name to its calibrated probability. Empty when the served model skipped the report, and always empty for non-staff users.
+ * * `scored` - scored
+ * * `stale` - stale
+ * * `unscored` - unscored
+ * * `not_available` - not_available
*/
- export type ReportRankingScores = {[key: string]: number};
+ export type RankingStatusEnum = typeof RankingStatusEnum[keyof typeof RankingStatusEnum];
+
+
+ export const RankingStatusEnum = {
+ Scored: 'scored',
+ Stale: 'stale',
+ Unscored: 'unscored',
+ NotAvailable: 'not_available',
+ } as const;
/**
- * Outcome head name to its probability divided by the head's training base rate, e.g. 2.7 means 2.7x as likely as the average report. A head without a saved base rate has no entry.
+ * * `open` - Open
+ * * `action` - Action
+ * * `fix` - Fix
*/
- export type ReportRankingLifts = {[key: string]: number};
+ export type RankingOutcomeEnum = typeof RankingOutcomeEnum[keyof typeof RankingOutcomeEnum];
+
+
+ export const RankingOutcomeEnum = {
+ Open: 'open',
+ Action: 'action',
+ Fix: 'fix',
+ } as const;
+
+ export interface ReportRankingPrediction {
+ /** Outcome the model predicts. Read inbox-ranking-outcomes-list for what each outcome means.
+ *
+ * * `open` - Open
+ * * `action` - Action
+ * * `fix` - Fix */
+ outcome: RankingOutcomeEnum;
+ /** The model's calibrated probability that the outcome happens within the outcome's horizon. */
+ probability: number;
+ /**
+ * The probability divided by the outcome's base rate. 1.0 is an average report, 2.7 means 2.7x as likely as the average report. Compare lifts across outcomes, not raw probabilities. Null when the model has no saved base rate for the outcome.
+ * @nullable
+ */
+ lift: number | null;
+ /** True when the training run could read the outcome's holdout AUC. Treat other outcomes with caution. */
+ readable: boolean;
+ }
export interface ReportRanking {
- /** Key of the served model in the scoring pass, as `<model_name>@<model_version>`. Empty for non-staff users. */
- served_key: string;
- /** Feature family of the served model. Empty for non-staff users. */
- model_name: string;
- /** Training partition of the served model, as `YYYY-MM-DD`. Empty for non-staff users. */
- model_version: string;
- /** Version of the serving manifest that chose the model. Empty for non-staff users. */
- manifest_version: string;
- /** When the scoring sweep scored the report. */
- scored_at: string;
- /** Outcome head name to its calibrated probability. Empty when the served model skipped the report, and always empty for non-staff users. */
- scores: ReportRankingScores;
- /** Outcome head name to its probability divided by the head's training base rate, e.g. 2.7 means 2.7x as likely as the average report. A head without a saved base rate has no entry. */
- lifts: ReportRankingLifts;
- /** Heads whose holdout AUC the training run could read. Treat scores of other heads with caution. */
- readable_heads: string[];
- /** True when the report's title or summary was edited after the text this score read. The score describes the old text: the inbox hides its lift and the model sort treats the report as unscored. */
- stale: boolean;
+ /** scored: the predictions describe the current text. stale: the title or summary changed after the text the model read, so the predictions describe the old text and the model sorts and filters treat the report as unscored. unscored: the model has not scored the report yet, or skipped it. This does not mean a low score. not_available: you cannot see the ranking model's output, so every other field is empty.
+ *
+ * * `scored` - scored
+ * * `stale` - stale
+ * * `unscored` - unscored
+ * * `not_available` - not_available */
+ status: RankingStatusEnum;
+ /**
+ * Key of the served model that wrote the score, as `<model_name>@<model_version>`. Do not compare predictions from different keys.
+ * @nullable
+ */
+ model: string | null;
+ /**
+ * When the scoring sweep scored the report.
+ * @nullable
+ */
+ scored_at: string | null;
+ /** The served model's predictions, one per public outcome that it scored. A prediction says what may happen, never what did: the report's status and its PR fields say whether it was resolved. */
+ predictions: ReportRankingPrediction[];
}
export interface ReportPlanStep {
@@ -78377,12 +78416,12 @@ export namespace Schemas {
* @nullable
*/
readonly channel_id: string | null;
- /** The served model's score from the latest ranking score artefact. Null for users who are neither staff nor in the inbox model sort rollout, and null when the report has no score. */
- readonly ranking: ReportRanking | null;
+ /** The served ranking model's predictions from the latest ranking score artefact. Its status says whether the report is scored, stale or unscored, or that the user cannot see the ranking model. */
+ readonly ranking: ReportRanking;
/** The plan this report leads or is a step of, with every step in it. A report that other reports link to with `part_of` is a plan. Null when the report is in no plan, and always null in list responses, because finding a plan's steps scans the project's report links. */
readonly plan: ReportPlan | null;
/**
- * Coarse sort key from the ranker named by the list response's ranker field, for sort=recommended only. Lower keys rank first. It is the score decile of the ranker's head (open, action or fixed), then the priority as a tiebreak. The priority ranker uses the priority only. Reports without a usable score sort after scored ones. Null for every other sort.
+ * Coarse sort key from the ranker named by the list response's ranker field, for sort=recommended only. Lower keys rank first. It is the score decile of the ranker's outcome (open, action or fix), then the priority as a tiebreak. The priority ranker uses the priority only. Reports without a usable score sort after scored ones. Null for every other sort.
* @nullable
*/
readonly inbox_merge_key: number | null;
@@ -78396,7 +78435,7 @@ export namespace Schemas {
previous?: string | null;
results: SignalReportList[];
/**
- * The ranker that ordered the page when sort=recommended: priority, or open, action or fixed for that head of the served ranking model. Null for every other sort. Each row's inbox_merge_key belongs to this ranker.
+ * The ranker that ordered the page when sort=recommended: priority, or open, action or fix for that outcome of the served ranking model. Null for every other sort. Each row's inbox_merge_key belongs to this ranker.
* @nullable
*/
ranker?: string | null;
@@ -99431,6 +99470,66 @@ export namespace Schemas {
Jev: 'jev',
} as const;
+ export interface RankingOutcomeInfo {
+ /** Name of the outcome in the `ranking` field's predictions.
+ *
+ * * `open` - Open
+ * * `action` - Action
+ * * `fix` - Fix */
+ outcome: RankingOutcomeEnum;
+ /** Short label the inbox shows for the outcome. */
+ label: string;
+ /** What the outcome predicts, in one sentence. */
+ description: string;
+ /**
+ * Days after scoring within which the outcome counts. Null when the served model did not save it.
+ * @nullable
+ */
+ horizon_days: number | null;
+ /**
+ * Name of the report set the outcome was trained on. Null when the served model did not save it.
+ * @nullable
+ */
+ cohort: string | null;
+ /**
+ * The outcome's base rate: a report's lift is its probability divided by this value. Null when the served model has no saved base rate for the outcome.
+ * @nullable
+ */
+ base_rate: number | null;
+ /** True when the training run could read the outcome's holdout AUC. Treat other outcomes with caution. */
+ readable: boolean;
+ }
+
+ /**
+ * * `scored` - scored
+ * * `unscored` - unscored
+ * * `not_available` - not_available
+ */
+ export type RankingOutcomesStatusEnum = typeof RankingOutcomesStatusEnum[keyof typeof RankingOutcomesStatusEnum];
+
+
+ export const RankingOutcomesStatusEnum = {
+ Scored: 'scored',
+ Unscored: 'unscored',
+ NotAvailable: 'not_available',
+ } as const;
+
+ export interface RankingOutcomesResponse {
+ /** scored: the served model has scored reports in this project. unscored: no readable score yet. not_available: you cannot see the ranking model's output.
+ *
+ * * `scored` - scored
+ * * `unscored` - unscored
+ * * `not_available` - not_available */
+ status: RankingOutcomesStatusEnum;
+ /**
+ * Key of the served model, as `<model_name>@<model_version>`, from the project's latest score.
+ * @nullable
+ */
+ model: string | null;
+ /** One entry per public outcome, in the inbox order. */
+ outcomes: RankingOutcomeInfo[];
+ }
+
export interface Rapid7InsightvmScannerFindingSignalExtra {
severity: string | null;
cvss_v3_score: string | null;
@@ -100522,8 +100621,8 @@ export namespace Schemas {
* @nullable
*/
readonly channel_id: string | null;
- /** The served model's score from the latest ranking score artefact. Null for users who are neither staff nor in the inbox model sort rollout, and null when the report has no score. */
- readonly ranking: ReportRanking | null;
+ /** The served ranking model's predictions from the latest ranking score artefact. Its status says whether the report is scored, stale or unscored, or that the user cannot see the ranking model. */
+ readonly ranking: ReportRanking;
/** The plan this report leads or is a step of, with every step in it. A report that other reports link to with `part_of` is a plan. Null when the report is in no plan, and always null in list responses, because finding a plan's steps scans the project's report links. */
readonly plan: ReportPlan | null;
}
@@ -136266,13 +136365,21 @@ export namespace Schemas {
*/
offset?: number;
/**
- * Comma-separated ordering clauses. Each clause is a field name optionally prefixed with '-' for descending. Allowed fields: status, is_suggested_reviewer, signal_count, total_weight, priority, created_at, updated_at, id, ranking_pr_merged, ranking_pr_created, ranking_action, ranking_open, ranking_fixed, ranking_discuss, ranking_thumbs_up, ranking_reviewer_fix, ranking_refund, ranking_dismiss_wrong, ranking_dismiss_lowvalue. Defaults to '-is_suggested_reviewer,status,-updated_at'. The ranking_* fields sort by the served ranking model's probability for that outcome head, with unscored reports last in either direction. They need staff or the inbox model sort rollout: other users get a 400.
+ * Comma-separated ordering clauses. Each clause is a field name optionally prefixed with '-' for descending. Allowed fields: status, is_suggested_reviewer, signal_count, total_weight, priority, created_at, updated_at, id, ranking_pr_merged, ranking_pr_created, ranking_action, ranking_open, ranking_fix, ranking_discuss, ranking_thumbs_up, ranking_reviewer_fix, ranking_refund, ranking_dismiss_wrong, ranking_dismiss_lowvalue. Defaults to '-is_suggested_reviewer,status,-updated_at'. The ranking_* fields sort by the served ranking model's probability for that outcome head, with unscored reports last in either direction. They need staff or the inbox model sort rollout: other users get a 400.
*/
ordering?: string;
/**
* Comma-separated list of priorities to include. Valid values: P0, P1, P2, P3, P4. Reports without a priority assignment are excluded when this filter is set.
*/
priority?: string;
+ /**
+ * Comma-separated <outcome>:<lift> pairs, e.g. fix:2,open:1.5. Keeps reports whose served ranking lift is at least the value for every named outcome. Outcomes: open, action, fix. Read inbox-ranking-outcomes-list for what each outcome means. Unscored and stale reports never match. Needs staff or the inbox model sort rollout: other users get a 400.
+ */
+ ranking_min_lift?: string;
+ /**
+ * Comma-separated <outcome>:<probability> pairs, e.g. fix:0.3. Keeps reports whose served ranking probability is at least the value for every named outcome. Outcomes: open, action, fix. Read inbox-ranking-outcomes-list for what each outcome means. Unscored and stale reports never match. Needs staff or the inbox model sort rollout: other users get a 400.
+ */
+ ranking_min_probability?: string;
/**
* Reviewer scope: for_me, entire_project, or teammate. Pass teammate_uuid with teammate.
*/
@@ -136290,7 +136397,7 @@ export namespace Schemas {
*/
search?: string;
/**
- * Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fixed head of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used.
+ * Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used.
*/
sort?: string;
/**
diff --git a/services/mcp/src/generated/signals/api.ts b/services/mcp/src/generated/signals/api.ts
index 666eef008d14..d703ce0ba7f7 100644
--- a/services/mcp/src/generated/signals/api.ts
+++ b/services/mcp/src/generated/signals/api.ts
@@ -3,7 +3,7 @@
* MCP service uses these Zod schemas for generated tool handlers.
* To regenerate: hogli build:openapi
*
- * PostHog API - MCP 71 enabled ops
+ * PostHog API - MCP 72 enabled ops
* OpenAPI spec version: 1.0.0
*/
import * as zod from 'zod'
@@ -264,7 +264,7 @@ export const SignalsReportsListQueryParams = () => zod.object({
.string()
.optional()
.describe(
- "Comma-separated ordering clauses. Each clause is a field name optionally prefixed with '-' for descending. Allowed fields: status, is_suggested_reviewer, signal_count, total_weight, priority, created_at, updated_at, id, ranking_pr_merged, ranking_pr_created, ranking_action, ranking_open, ranking_fixed, ranking_discuss, ranking_thumbs_up, ranking_reviewer_fix, ranking_refund, ranking_dismiss_wrong, ranking_dismiss_lowvalue. Defaults to '-is_suggested_reviewer,status,-updated_at'. The ranking_\* fields sort by the served ranking model's probability for that outcome head, with unscored reports last in either direction. They need staff or the inbox model sort rollout: other users get a 400."
+ "Comma-separated ordering clauses. Each clause is a field name optionally prefixed with '-' for descending. Allowed fields: status, is_suggested_reviewer, signal_count, total_weight, priority, created_at, updated_at, id, ranking_pr_merged, ranking_pr_created, ranking_action, ranking_open, ranking_fix, ranking_discuss, ranking_thumbs_up, ranking_reviewer_fix, ranking_refund, ranking_dismiss_wrong, ranking_dismiss_lowvalue. Defaults to '-is_suggested_reviewer,status,-updated_at'. The ranking_\* fields sort by the served ranking model's probability for that outcome head, with unscored reports last in either direction. They need staff or the inbox model sort rollout: other users get a 400."
),
priority: zod
.string()
@@ -272,6 +272,18 @@ export const SignalsReportsListQueryParams = () => zod.object({
.describe(
'Comma-separated list of priorities to include. Valid values: P0, P1, P2, P3, P4. Reports without a priority assignment are excluded when this filter is set.'
),
+ ranking_min_lift: zod
+ .string()
+ .optional()
+ .describe(
+ 'Comma-separated <outcome>:<lift> pairs, e.g. fix:2,open:1.5. Keeps reports whose served ranking lift is at least the value for every named outcome. Outcomes: open, action, fix. Read inbox-ranking-outcomes-list for what each outcome means. Unscored and stale reports never match. Needs staff or the inbox model sort rollout: other users get a 400.'
+ ),
+ ranking_min_probability: zod
+ .string()
+ .optional()
+ .describe(
+ 'Comma-separated <outcome>:<probability> pairs, e.g. fix:0.3. Keeps reports whose served ranking probability is at least the value for every named outcome. Outcomes: open, action, fix. Read inbox-ranking-outcomes-list for what each outcome means. Unscored and stale reports never match. Needs staff or the inbox model sort rollout: other users get a 400.'
+ ),
scope: zod
.string()
.optional()
@@ -293,7 +305,7 @@ export const SignalsReportsListQueryParams = () => zod.object({
.string()
.optional()
.describe(
- "Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fixed head of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used."
+ "Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used."
),
source_id: zod
.string()
@@ -1228,6 +1240,18 @@ export const SignalsReportsBulkStateCreateBody = () => zod.object({
),
})
+/**
+ * What each public outcome of the served ranking model predicts, with its horizon, cohort, base rate and readability from the model that scored this project's reports. Users who cannot see the ranking model get status not_available and no outcomes.
+ * @summary List the ranking model's public outcomes
+ */
+export const SignalsReportsRankingOutcomesListParams = () => zod.object({
+ project_id: zod
+ .string()
+ .describe(
+ "Project ID of the project you're trying to access. To find the ID of the project, make a call to \/api\/projects\/."
+ ),
+})
+
/**
* Create a scout skill and its runnable config atomically. Give it a `display_name` — the label people read, kept exactly as written — and the scout's permanent skill name is generated from it, with a numeric suffix when that name is taken, so two scouts may share a label without sharing an identity. Pass `name` instead to pick that identifier yourself; any valid skill name works, since the config row is what makes a skill a scout. The skill always receives the report-channel tools. The optional config controls schedule, enablement, dry-run posture, network access, and typed destinations such as Slack. Repeating the same definition is safe and applies any supplied config fields; reusing an explicit `name` for a different definition returns 409.
* @summary Create a scout
diff --git a/services/mcp/src/tools/generated/signals.ts b/services/mcp/src/tools/generated/signals.ts
index 7f31fd6b3a44..9b5d0ba93f29 100644
--- a/services/mcp/src/tools/generated/signals.ts
+++ b/services/mcp/src/tools/generated/signals.ts
@@ -57,6 +57,24 @@ const activitySummaryVersionsList = (): ToolBase<
},
})
+const InboxRankingOutcomesListSchema = () => z.object({})
+
+const inboxRankingOutcomesList = (): ToolBase<
+ ReturnType<typeof InboxRankingOutcomesListSchema>,
+ Schemas.RankingOutcomesResponse
+> => ({
+ name: 'inbox-ranking-outcomes-list',
+ schema: InboxRankingOutcomesListSchema(),
+ handler: async (context: Context, _params: z.infer<ReturnType<typeof InboxRankingOutcomesListSchema>>) => {
+ const projectId = await context.stateManager.getProjectId()
+ const result = await context.api.request<Schemas.RankingOutcomesResponse>({
+ method: 'GET',
+ path: `/api/projects/${encodeURIComponent(String(projectId))}/signals/reports/ranking_outcomes/`,
+ })
+ return result
+ },
+})
+
const InboxReportActionsListSchema = () => {
const SignalsReportsActionsListParams = orvalSchemas.SignalsReportsActionsListParams()
return z.preprocess(
@@ -602,6 +620,8 @@ const inboxReportsList = (): ToolBase<
offset: params.offset,
ordering: params.ordering,
priority: params.priority,
+ ranking_min_lift: params.ranking_min_lift,
+ ranking_min_probability: params.ranking_min_probability,
scope: params.scope,
scout: params.scout,
scout_prefix: params.scout_prefix,
@@ -649,6 +669,9 @@ const inboxReportsList = (): ToolBase<
'tracker_issue_error',
'work_state',
'assignee',
+ 'ranking.status',
+ 'ranking.scored_at',
+ 'ranking.predictions',
'created_at',
'updated_at',
])
@@ -3322,6 +3345,7 @@ const signalsScoutScratchpadSearch = (): ToolBase<
export const GENERATED_TOOLS: Record<string, () => ToolBase<ZodObjectAny>> = {
'activity-summary-get': activitySummaryGet,
'activity-summary-versions-list': activitySummaryVersionsList,
+ 'inbox-ranking-outcomes-list': inboxRankingOutcomesList,
'inbox-report-actions-list': inboxReportActionsList,
'inbox-report-actions-run': inboxReportActionsRun,
'inbox-report-artefacts-create': inboxReportArtefactsCreate,
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-ranking-outcomes-list.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-ranking-outcomes-list.json
new file mode 100644
index 000000000000..7b7ea9186078
--- /dev/null
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-ranking-outcomes-list.json
@@ -0,0 +1,5 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "properties": {},
+ "type": "object"
+}
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json
index 9426b78f485e..9daf00f0300a 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/inbox-reports-list.json
@@ -64,13 +64,21 @@
"type": "number"
},
"ordering": {
- "description": "Comma-separated ordering clauses. Each clause is a field name optionally prefixed with '-' for descending. Allowed fields: status, is_suggested_reviewer, signal_count, total_weight, priority, created_at, updated_at, id, ranking_pr_merged, ranking_pr_created, ranking_action, ranking_open, ranking_fixed, ranking_discuss, ranking_thumbs_up, ranking_reviewer_fix, ranking_refund, ranking_dismiss_wrong, ranking_dismiss_lowvalue. Defaults to '-is_suggested_reviewer,status,-updated_at'. The ranking_* fields sort by the served ranking model's probability for that outcome head, with unscored reports last in either direction. They need staff or the inbox model sort rollout: other users get a 400.",
+ "description": "Comma-separated ordering clauses. Each clause is a field name optionally prefixed with '-' for descending. Allowed fields: status, is_suggested_reviewer, signal_count, total_weight, priority, created_at, updated_at, id, ranking_pr_merged, ranking_pr_created, ranking_action, ranking_open, ranking_fix, ranking_discuss, ranking_thumbs_up, ranking_reviewer_fix, ranking_refund, ranking_dismiss_wrong, ranking_dismiss_lowvalue. Defaults to '-is_suggested_reviewer,status,-updated_at'. The ranking_* fields sort by the served ranking model's probability for that outcome head, with unscored reports last in either direction. They need staff or the inbox model sort rollout: other users get a 400.",
"type": "string"
},
"priority": {
"description": "Comma-separated list of priorities to include. Valid values: P0, P1, P2, P3, P4. Reports without a priority assignment are excluded when this filter is set.",
"type": "string"
},
+ "ranking_min_lift": {
+ "description": "Comma-separated <outcome>:<lift> pairs, e.g. fix:2,open:1.5. Keeps reports whose served ranking lift is at least the value for every named outcome. Outcomes: open, action, fix. Read inbox-ranking-outcomes-list for what each outcome means. Unscored and stale reports never match. Needs staff or the inbox model sort rollout: other users get a 400.",
+ "type": "string"
+ },
+ "ranking_min_probability": {
+ "description": "Comma-separated <outcome>:<probability> pairs, e.g. fix:0.3. Keeps reports whose served ranking probability is at least the value for every named outcome. Outcomes: open, action, fix. Read inbox-ranking-outcomes-list for what each outcome means. Unscored and stale reports never match. Needs staff or the inbox model sort rollout: other users get a 400.",
+ "type": "string"
+ },
"scope": {
"description": "Reviewer scope: for_me, entire_project, or teammate. Pass teammate_uuid with teammate.",
"type": "string"
@@ -88,7 +96,7 @@
"type": "string"
},
"sort": {
- "description": "Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fixed head of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used.",
+ "description": "Inbox sort preset: priority, recommended, last_updated, newest, or oldest. Ignored when ordering is supplied. recommended resolves per user to a ranker: the priority order, or the open, action or fix outcome of the served ranking model with unscored and stale reports in priority order after the scored ones. The response's ranker field names the one used.",
"type": "string"
},
"source_id": {