Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): rescore inbox reports only when the served key changes (#124504)

posthog[bot]d5d80379eb58
diff --git a/docs/internal/inbox-ranking-training.md b/docs/internal/inbox-ranking-training.md
index 73be4d0cb4cb..7620b3966c4e 100644
--- a/docs/internal/inbox-ranking-training.md
+++ b/docs/internal/inbox-ranking-training.md
@@ -200,7 +200,7 @@ A missing object is a skip in the asset metadata, not a failure.
Caveats:
- A manual promotion can change the served model part of the way through D, so one day's cohort can split across two versions, and one report can hold a row for each. Grades stay per `(model_name, model_version, model_role)`. Never pool them across versions.
-- The sweep scores a candidate only after a manifest names it, so a candidate's online cohort starts the day after it was trained. `pool_coverage_by_model` shows how much of the pool each key covered.
+- The sweep scores a candidate only after a manifest names it, so a candidate's online cohort starts the day after it was trained. A new manifest does not rescore reports that already have a score. Only a change of the served key does that. So a candidate scores only the reports that are new or have a new vector after its manifest is published. `pool_coverage_by_model` shows how much of the pool each key covered.
- A report first scored after D ends, for example when its vector arrived late, is not in D's rows. `served_pool_coverage` shows this for the served model. A later score never fills it in.
- The sweep scores with the vector current at scoring time, so two keys that scored one report at different times did not always read the same vector.
- A deployment where the sweep is off writes an empty object with coverage 0 and grades nothing.
diff --git a/products/signals/backend/ranking/sweep.py b/products/signals/backend/ranking/sweep.py
index 1db1e7639911..40b387c3dcc7 100644
--- a/products/signals/backend/ranking/sweep.py
+++ b/products/signals/backend/ranking/sweep.py
@@ -1,9 +1,14 @@
"""The scoring sweep: a scheduled Temporal workflow that scores the inbox reports that are due.
-A report is due when its score is missing or out of date. The served model reads only the report's
-vector, so the vector log is the change signal, not `SignalReport.updated_at`: a scout edit saves
-`update_fields=["title", "summary"]` and leaves `updated_at` alone, and the grouping pipeline
-moves `updated_at` with no text change. A new vector lands only when a rendering's text changes.
+A report is due when its score is missing, older than its vector, or from another served model. The
+served model reads only the report's vector, so the vector log is the change signal, not
+`SignalReport.updated_at`: a scout edit saves `update_fields=["title", "summary"]` and leaves
+`updated_at` alone, and the grouping pipeline moves `updated_at` with no text change. A new vector
+lands only when a rendering's text changes.
+
+A new manifest alone does not make a report due. Training publishes one every night because the
+challenger models change, and a rescore of the whole window for each one makes new reports wait.
+A challenger scores only the reports that become due after a manifest names it.
Nothing reads the `ranking_score` artefacts to order the inbox yet.
"""
@@ -121,7 +126,6 @@ class _ScoreStamp(pydantic.BaseModel):
scored_at: datetime.datetime
embedding_inserted_at: datetime.datetime | None = None
- manifest_version: str
served_key: str
@@ -211,24 +215,20 @@ def _latest_scores(report_ids_by_team: Mapping[int, Sequence[str]]) -> dict[str,
return scores
-def _is_due(
- score: _ScoreStamp | None, vector_inserted_at: datetime.datetime, manifest_version: str, served_key: str
-) -> bool:
- # A `served` override changes the served model and leaves `manifest_version` as it is, so the
- # served key is compared too. Setting or removing an override then rescores every report.
+def _is_due(score: _ScoreStamp | None, vector_inserted_at: datetime.datetime, served_key: str) -> bool:
+ # A promotion or a `served` override changes the served key, so each one rescores every report.
return (
score is None
or score.embedding_inserted_at is None
or vector_inserted_at > score.embedding_inserted_at
- or score.manifest_version != manifest_version
or score.served_key != served_key
)
def reports_due_for_scoring(
- now: datetime.datetime, *, manifest_version: str, served_key: str, rendering: str, limit: int
+ now: datetime.datetime, *, served_key: str, rendering: str, limit: int
) -> list[ScoringCandidate]:
- """In-window scorable reports whose score is missing, older than their vector, or from another manifest or served model.
+ """In-window scorable reports whose score is missing, older than their vector, or from another served model.
Unscored reports come first, then the oldest scores, so no report starves under the cap. A
report with no live vector is never a candidate, so a safety-suppressed or retracted report is
@@ -247,7 +247,7 @@ def reports_due_for_scoring(
for report_id in kept:
team_id, vector_inserted_at = vectors[report_id]
score = latest.get(report_id)
- if not _is_due(score, vector_inserted_at, manifest_version, served_key):
+ if not _is_due(score, vector_inserted_at, served_key):
continue
candidate = ScoringCandidate(team_id=team_id, report_id=report_id, vector_inserted_at=vector_inserted_at)
due.append((score.scored_at if score else None, candidate))
@@ -288,7 +288,6 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult:
override = serving.served_override
candidates = reports_due_for_scoring(
now,
- manifest_version=manifest_version,
served_key=serving.manifest.served.key,
rendering=scorer.served_rendering(serving),
limit=limit or settings.INBOX_RANKING_SCORING_MAX_REPORTS_PER_TICK,
diff --git a/products/signals/backend/test/test_ranking_sweep.py b/products/signals/backend/test/test_ranking_sweep.py
index fbf8535d50a8..082e21d7a89b 100644
--- a/products/signals/backend/test/test_ranking_sweep.py
+++ b/products/signals/backend/test/test_ranking_sweep.py
@@ -30,6 +30,7 @@ MANIFEST = "manifest-b"
OLD_MANIFEST = "manifest-a"
SERVED_VERSION = "2026-09-01"
SERVED_KEY = f"report_embeddings@{SERVED_VERSION}"
+OLD_SERVED_VERSION = "2026-08-25"
def _score(
@@ -133,7 +134,6 @@ class TestReportsDueForScoring(ClickhouseTestMixin, BaseTest):
def _due(self, limit: int = 100) -> list[str]:
candidates = reports_due_for_scoring(
self.now,
- manifest_version=MANIFEST,
served_key=SERVED_KEY,
rendering=EMBEDDING_RENDERING_TITLE_SUMMARY,
limit=limit,
@@ -156,12 +156,13 @@ class TestReportsDueForScoring(ClickhouseTestMixin, BaseTest):
self._vector(edited, hours_ago=2)
assert SignalReport.objects.get(id=edited).updated_at == updated_at
+ # A manifest that changes only the challengers keeps the served key, so the score stays current.
other_manifest = self._report()
self._scored(other_manifest, vector_at=self._vector(other_manifest, hours_ago=5), manifest=OLD_MANIFEST)
# A served override changes the served model under the same manifest version.
other_served = self._report()
- self._scored(other_served, vector_at=self._vector(other_served, hours_ago=5), served_version="2026-08-25")
+ self._scored(other_served, vector_at=self._vector(other_served, hours_ago=5), served_version=OLD_SERVED_VERSION)
retracted = self._report()
self._vector(retracted, hours_ago=5)
@@ -180,19 +181,24 @@ class TestReportsDueForScoring(ClickhouseTestMixin, BaseTest):
# A newer current score stored under another team must not count as this report's score.
cross_team_score = self._report()
cross_team_vector = self._vector(cross_team_score, hours_ago=5)
- self._scored(cross_team_score, vector_at=cross_team_vector, manifest=OLD_MANIFEST, hours_ago=3)
+ self._scored(cross_team_score, vector_at=cross_team_vector, served_version=OLD_SERVED_VERSION, hours_ago=3)
self._scored(cross_team_score, vector_at=cross_team_vector, team=other_team)
SignalReportArtefact.objects.filter(team=other_team, report_id=cross_team_score).update(
created_at=self.now + datetime.timedelta(hours=1)
)
- assert sorted(self._due()) == sorted([unscored, edited, other_manifest, other_served, cross_team_score])
+ assert sorted(self._due()) == sorted([unscored, edited, other_served, cross_team_score])
def test_the_cap_keeps_unscored_reports_first_then_the_oldest_scores(self) -> None:
recently_scored, long_ago_scored, unscored = self._report(), self._report(), self._report()
- self._scored(recently_scored, vector_at=self._vector(recently_scored, hours_ago=5), manifest=OLD_MANIFEST)
self._scored(
- long_ago_scored, vector_at=self._vector(long_ago_scored, hours_ago=5), manifest=OLD_MANIFEST, hours_ago=3
+ recently_scored, vector_at=self._vector(recently_scored, hours_ago=5), served_version=OLD_SERVED_VERSION
+ )
+ self._scored(
+ long_ago_scored,
+ vector_at=self._vector(long_ago_scored, hours_ago=5),
+ served_version=OLD_SERVED_VERSION,
+ hours_ago=3,
)
self._vector(unscored, hours_ago=5)