Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): record which inputs changed when a personal ranking rebuilds (#129353)

posthog[bot]d64403bf2a76
diff --git a/products/signals/backend/migrations/0233_personalinboxranking_input_parts.py b/products/signals/backend/migrations/0233_personalinboxranking_input_parts.py
new file mode 100644
index 000000000000..04719655616c
--- /dev/null
+++ b/products/signals/backend/migrations/0233_personalinboxranking_input_parts.py
@@ -0,0 +1,13 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [("signals", "0232_signalreport_created_by_index")]
+
+ operations = [
+ migrations.AddField(
+ model_name="personalinboxranking",
+ name="input_parts",
+ field=models.JSONField(blank=True, null=True),
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index 5ce15909f164..ae6c210e7273 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0232_signalreport_created_by_index
+0233_personalinboxranking_input_parts
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index 79ea8a9bba1e..5461576566e3 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -4372,6 +4372,8 @@ class PersonalInboxRanking(TeamScopedRootMixin, UUIDModel):
# Covers the candidates with their status, priority, PR state, needs-you facts and score, the summary version, the prompt version and the focus.
input_hash = models.CharField(max_length=64)
+ # The inputs of `input_hash` as one hash per part, from `input_parts`. Null on rows from before the parts.
+ input_parts = models.JSONField(null=True, blank=True)
# Report ids in candidate order.
candidate_ids = models.JSONField(default=list)
# Every candidate with its rank, tier and reason, and the folded reports. See `PersonalRankingResult.as_json`.
diff --git a/products/signals/backend/personal_ranking/logic.py b/products/signals/backend/personal_ranking/logic.py
index 9ea94dcc8fc7..0378fd8c987d 100644
--- a/products/signals/backend/personal_ranking/logic.py
+++ b/products/signals/backend/personal_ranking/logic.py
@@ -71,6 +71,15 @@ RETRYABLE_FALLBACK_REASONS = frozenset(
{FALLBACK_LLM_ERROR, FALLBACK_TIMEOUT, FALLBACK_MALFORMED, FALLBACK_TRUNCATED, FALLBACK_AI_NOT_APPROVED}
)
+# The input parts that `input_parts` hashes one by one. The report parts have one hash for each candidate.
+REPORT_PARTS = ("text", "status", "priority", "pull_request_state", "needs_you", "scores")
+INPUT_PARTS = ("candidate_set", *REPORT_PARTS, "activity_summary_version", "prompt_version", "focus")
+# The person had no list before this build.
+CHANGED_NO_PREVIOUS = "no_previous_list"
+# The previous list has no part hashes, so the build cannot tell which part changed.
+CHANGED_UNKNOWN = "unknown"
+_PART_HASH_CHARS = 16
+
BuildStatus = Literal["ok", "fallback", "empty", "disabled"]
# What started a build, on the build event and the trace, so cost can be split by it.
BuildTrigger = Literal["on_demand", "tick"]
@@ -92,6 +101,32 @@ class _ModelCall:
fallback_reason: str | None
+@frozen
+class InputChanges:
+ """How the inputs of a build differ from the inputs of the person's previous list."""
+
+ # Names from `INPUT_PARTS`, or only `CHANGED_NO_PREVIOUS` or `CHANGED_UNKNOWN`.
+ changed_parts: tuple[str, ...]
+ # The reports in both lists with a changed report part. Null when the parts are not comparable.
+ changed_report_count: int | None = None
+ # The largest probability move of a report in both lists. Null when no probability is in both lists.
+ max_score_move: float | None = None
+ joined_count: int | None = None
+ left_count: int | None = None
+ # The reports that left and were in the first `rank_top` entries of the previous list.
+ left_top_count: int | None = None
+
+ def properties(self) -> dict[str, Any]:
+ return {
+ "changed_parts": list(self.changed_parts),
+ "changed_report_count": self.changed_report_count,
+ "max_score_move": self.max_score_move,
+ "candidates_joined": self.joined_count,
+ "candidates_left": self.left_count,
+ "candidates_left_top": self.left_top_count,
+ }
+
+
@frozen
class PersonalRankingBuild:
status: BuildStatus
@@ -111,6 +146,10 @@ class PersonalRankingBuild:
latency_ms: int | None = None
# None for a dry run or a disabled build, which store nothing.
ranking_id: str | None = None
+ # The stored `input_parts`. None for a disabled build.
+ input_parts: dict[str, Any] | None = None
+ # None for a dry run or a disabled build.
+ input_changes: InputChanges | None = None
def result_json(self) -> dict[str, Any]:
"""The stored `result`: every candidate by report id, with its rank, tier and reason."""
@@ -187,6 +226,104 @@ def input_hash(candidates: list[Candidate], *, activity_summary_version: int | N
return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode()).hexdigest()
+def _part_hash(value: Any) -> str:
+ return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode()).hexdigest()[:_PART_HASH_CHARS]
+
+
+def _report_parts(candidate: Candidate) -> dict[str, Any]:
+ predictions = candidate.predictions
+ return {
+ "text": [candidate.title, candidate.summary_lead],
+ "status": candidate.status,
+ "priority": candidate.priority,
+ "pull_request_state": candidate.pull_request_state,
+ "needs_you": list(candidate.needs_you),
+ "scores": [
+ predictions.status if predictions else None,
+ [[entry.outcome, round(entry.probability, 2)] for entry in predictions.predictions] if predictions else [],
+ ],
+ }
+
+
+def input_parts(
+ candidates: list[Candidate], *, activity_summary_version: int | None, focus: str | None
+) -> dict[str, Any]:
+ """The inputs of `input_hash` as one hash per part, so a later build can name the parts that changed.
+
+ The report parts have one hash for each report. `scores` keeps the probabilities, so a later build can measure the move.
+ """
+ return {
+ "parts": {
+ "activity_summary_version": _part_hash(activity_summary_version),
+ "prompt_version": _part_hash(PROMPT_VERSION),
+ "focus": _part_hash(focus),
+ },
+ "reports": {
+ candidate.report_id: {part: _part_hash(value) for part, value in _report_parts(candidate).items()}
+ for candidate in candidates
+ },
+ "scores": {
+ candidate.report_id: {
+ entry.outcome: round(entry.probability, 4) for entry in candidate.predictions.predictions
+ }
+ for candidate in candidates
+ if candidate.predictions
+ },
+ }
+
+
+def _max_score_move(previous: dict[str, dict[str, float]], current: dict[str, dict[str, float]]) -> float | None:
+ moves = [
+ abs(probability - previous[report_id][outcome])
+ for report_id, scores in current.items()
+ if report_id in previous
+ for outcome, probability in scores.items()
+ if outcome in previous[report_id]
+ ]
+ return round(max(moves), 4) if moves else None
+
+
+def compare_inputs(previous: PersonalInboxRanking | None, current: dict[str, Any], *, rank_top: int) -> InputChanges:
+ """The parts that differ between the previous list and the current inputs. Never raises."""
+ if previous is None:
+ return InputChanges(changed_parts=(CHANGED_NO_PREVIOUS,))
+ stored = previous.input_parts
+ try:
+ if not stored:
+ return InputChanges(changed_parts=(CHANGED_UNKNOWN,))
+ stored_reports: dict[str, dict[str, str]] = stored["reports"]
+ current_reports: dict[str, dict[str, str]] = current["reports"]
+ joined = current_reports.keys() - stored_reports.keys()
+ left = stored_reports.keys() - current_reports.keys()
+ changed = {"candidate_set"} if joined or left else set()
+ changed |= {part for part in current["parts"] if current["parts"][part] != stored["parts"].get(part)}
+ changed_reports = 0
+ for report_id in current_reports.keys() & stored_reports.keys():
+ report_changed = {
+ part
+ for part in REPORT_PARTS
+ if current_reports[report_id].get(part) != stored_reports[report_id].get(part)
+ }
+ changed |= report_changed
+ changed_reports += bool(report_changed)
+ top_ids = {entry["report_id"] for entry in previous.result.get("ranked", [])[:rank_top]}
+ return InputChanges(
+ changed_parts=tuple(part for part in INPUT_PARTS if part in changed),
+ changed_report_count=changed_reports,
+ max_score_move=_max_score_move(stored.get("scores", {}), current["scores"]),
+ joined_count=len(joined),
+ left_count=len(left),
+ left_top_count=len(left & top_ids),
+ )
+ except (KeyError, TypeError, AttributeError):
+ logger.warning("signals.personal_ranking.unreadable_input_parts", ranking_id=str(previous.id))
+ return InputChanges(changed_parts=(CHANGED_UNKNOWN,))
+
+
+def latest_row(team: Team, user: User) -> PersonalInboxRanking | None:
+ return PersonalInboxRanking.objects.for_team(team.id).filter(user_id=user.id).order_by("-created_at", "-id").first()
+
+
def _client(
team: Team, user: User, now: datetime.datetime, config: PersonalRankingConfig, trigger: BuildTrigger
) -> OpenAI:
@@ -317,6 +454,7 @@ def _capture(
"tokens_in": build.tokens_in,
"tokens_out": build.tokens_out,
"latency_ms": build.latency_ms,
+ **(build.input_changes.properties() if build.input_changes is not None else {}),
},
groups={"organization": str(team.organization_id), "project": str(team.uuid)},
)
@@ -330,6 +468,7 @@ def _store(team: Team, user: User, build: PersonalRankingBuild) -> PersonalRanki
team_id=team.id,
user_id=user.id,
input_hash=build.input_hash or "",
+ input_parts=build.input_parts,
candidate_ids=[candidate.report_id for candidate in build.candidates],
result=build.result_json(),
personalized=build.personalized,
@@ -437,6 +576,7 @@ def build_personal_ranking(
fallback_reason = FALLBACK_MALFORMED
status = "ok" if validated is not None else "fallback"
+ parts = input_parts(candidates, activity_summary_version=summary_version, focus=focus)
build = PersonalRankingBuild(
status=status,
personalized=validated is not None,
@@ -449,6 +589,7 @@ def build_personal_ranking(
counts=validated.counts if validated is not None else ValidationCounts(),
limit=limit,
input_hash=input_hash(candidates, activity_summary_version=summary_version, focus=focus),
+ input_parts=parts,
activity_summary_version=summary_version,
model=config.model,
model_called=call is not None,
@@ -458,7 +599,8 @@ def build_personal_ranking(
)
if dry_run:
return build
- build = _store(team, user, build)
+ changes = compare_inputs(latest_row(team, user), parts, rank_top=config.rank_top)
+ build = _store(team, user, dataclasses.replace(build, input_changes=changes))
try:
_prune(team, user, config.max_rows_per_person)
except Exception:
diff --git a/products/signals/backend/personal_ranking/serve.py b/products/signals/backend/personal_ranking/serve.py
index 6ae4b7935646..76ea0557dc56 100644
--- a/products/signals/backend/personal_ranking/serve.py
+++ b/products/signals/backend/personal_ranking/serve.py
@@ -38,6 +38,7 @@ from products.signals.backend.personal_ranking.candidates import Candidate, gath
from products.signals.backend.personal_ranking.config import PersonalRankingConfig, read_config
from products.signals.backend.personal_ranking.logic import (
input_hash,
+ latest_row,
may_call_model,
model_order_reason,
normalize_focus,
@@ -216,10 +217,6 @@ def _start_build(
return "failed"
-def latest_row(team: Team, user: User) -> PersonalInboxRanking | None:
- return PersonalInboxRanking.objects.for_team(team.id).filter(user_id=user.id).order_by("-created_at", "-id").first()
-
-
def _row_stored_since(
team: Team, user: User, seen: PersonalInboxRanking | None, current_hash: str
) -> PersonalInboxRanking | None:
diff --git a/products/signals/backend/personal_ranking/tick.py b/products/signals/backend/personal_ranking/tick.py
index e5872660a8b9..8268114ec26a 100644
--- a/products/signals/backend/personal_ranking/tick.py
+++ b/products/signals/backend/personal_ranking/tick.py
@@ -50,13 +50,17 @@ from products.signals.backend.personal_ranking.config import (
read_config,
)
from products.signals.backend.personal_ranking.logic import (
+ CHANGED_NO_PREVIOUS,
+ CHANGED_UNKNOWN,
+ INPUT_PARTS,
PIPELINE_DISTINCT_ID,
RETRYABLE_FALLBACK_REASONS,
build_personal_ranking,
input_hash,
+ latest_row,
may_call_model,
)
-from products.signals.backend.personal_ranking.serve import latest_row, ready_summary_version
+from products.signals.backend.personal_ranking.serve import ready_summary_version
from products.signals.backend.personal_ranking_tasks import BUILD_LOCK_TTL_SECONDS, build_lock_key
logger = structlog.get_logger(__name__)
@@ -160,6 +164,8 @@ class PersonalRankingTickPlan:
class PersonalRankingPairResult:
status: PairStatus
model_called: bool = False
+ # The `changed_parts` of the build. Empty when the pair did not build.
+ changed_parts: list[str] = dataclasses.field(default_factory=list)
@frozen
@@ -169,6 +175,8 @@ class PersonalRankingTickResult:
due_without_list: int = 0
picked: int = 0
statuses: dict[str, int] = dataclasses.field(default_factory=dict)
+ # Built pairs by changed input part. A pair with two changed parts counts in both.
+ changed_parts: dict[str, int] = dataclasses.field(default_factory=dict)
errored: int = 0
stop_reason: str | None = None
oldest_due_hours: float | None = None
@@ -373,7 +381,11 @@ def _check_and_build(team: Team, user: User, config: PersonalRankingConfig) -> P
if build.status == "disabled":
return PersonalRankingPairResult(status="disabled")
failed = build.model_called and build.status == "fallback"
- return PersonalRankingPairResult(status="failed" if failed else "built", model_called=build.model_called)
+ return PersonalRankingPairResult(
+ status="failed" if failed else "built",
+ model_called=build.model_called,
+ changed_parts=list(build.input_changes.changed_parts) if build.input_changes is not None else [],
+ )
def run_pair(pair: TickPair) -> PersonalRankingPairResult:
@@ -399,6 +411,10 @@ def capture_tick(result: PersonalRankingTickResult) -> None:
"due_without_list": result.due_without_list,
"picked": result.picked,
**{status: result.statuses.get(status, 0) for status in PAIR_STATUSES},
+ **{
+ f"changed_{part}": result.changed_parts.get(part, 0)
+ for part in (*INPUT_PARTS, CHANGED_NO_PREVIOUS, CHANGED_UNKNOWN)
+ },
"errored": result.errored,
"stop_reason": result.stop_reason,
"oldest_due_hours": result.oldest_due_hours,
@@ -450,6 +466,7 @@ class PersonalRankingTickWorkflow:
deadline = started + datetime.timedelta(seconds=plan.time_budget_seconds)
statuses: dict[str, int] = {}
+ changed_parts: dict[str, int] = {}
errored = 0
# Model calls of this tick and the calls among them that failed or raised, for the circuit breaker.
calls = 0
@@ -491,6 +508,8 @@ class PersonalRankingTickWorkflow:
failures += 1
continue
statuses[outcome.status] = statuses.get(outcome.status, 0) + 1
+ for part in outcome.changed_parts:
+ changed_parts[part] = changed_parts.get(part, 0) + 1
calls += outcome.model_called
failures += outcome.status == "failed"
limit_stop = limit_stop or _LIMIT_STOPS.get(outcome.status)
@@ -504,6 +523,7 @@ class PersonalRankingTickWorkflow:
due_without_list=plan.due_without_list,
picked=picked,
statuses=statuses,
+ changed_parts=changed_parts,
errored=errored,
stop_reason=stop_reason,
oldest_due_hours=plan.oldest_due_hours,
diff --git a/products/signals/backend/test/test_personal_ranking.py b/products/signals/backend/test/test_personal_ranking.py
index d2e8a1fd5d35..e65ec88304f8 100644
--- a/products/signals/backend/test/test_personal_ranking.py
+++ b/products/signals/backend/test/test_personal_ranking.py
@@ -44,7 +44,14 @@ from products.signals.backend.personal_ranking.config import (
ReasonLimits,
parse_config,
)
-from products.signals.backend.personal_ranking.logic import BUILT_EVENT, build_personal_ranking
+from products.signals.backend.personal_ranking.logic import (
+ BUILT_EVENT,
+ InputChanges,
+ build_personal_ranking,
+ compare_inputs,
+ input_hash,
+ input_parts,
+)
from products.signals.backend.personal_ranking.prompt import render_person, system_prompt, user_message
from products.signals.backend.personal_ranking.validation import MalformedRankingError, validate_ranking
from products.signals.backend.personal_ranking_tasks import build_lock_key, build_personal_inbox_ranking
@@ -263,6 +270,103 @@ def _candidate(index: int, report_id: str, **overrides: Any) -> Candidate:
return Candidate(**{**fields, **overrides})
+_IDS = ["018f3d0e-0000-7000-8000-000000000001", "018f3d0e-0000-7000-8000-000000000002"]
+
+
+def _scored(probability: float) -> ReportPredictions:
+ return ReportPredictions(
+ status="scored",
+ model="m@1",
+ predictions=(OutcomePrediction(outcome="action", probability=probability, lift=None, readable=True),),
+ )
+
+
+class TestInputParts(SimpleTestCase):
+ def test_input_hash_keeps_its_value(self) -> None:
+ candidates = [_candidate(1, _IDS[0]), _candidate(2, _IDS[1], predictions=None, needs_you=())]
+
+ assert (
+ input_hash(candidates, activity_summary_version=7, focus="checkout")
+ == "8b3bc5b28875163b6769d657f08d4351f846fbd0293ca46563bd930743ed5958"
+ )
+
+ @parameterized.expand(
+ [
+ ("nothing", {}, 7, InputChanges(changed_parts=(), changed_report_count=0, max_score_move=0.0)),
+ (
+ "score_jitter_under_the_rounding",
+ {"predictions": _scored(0.421)},
+ 7,
+ InputChanges(changed_parts=(), changed_report_count=0, max_score_move=0.001),
+ ),
+ (
+ "score",
+ {"predictions": _scored(0.5)},
+ 7,
+ InputChanges(changed_parts=("scores",), changed_report_count=1, max_score_move=0.08),
+ ),
+ (
+ "summary_version",
+ {},
+ 8,
+ InputChanges(changed_parts=("activity_summary_version",), changed_report_count=0, max_score_move=0.0),
+ ),
+ (
+ "text_and_needs_you",
+ {"title": "Renamed", "needs_you": ("claimed",)},
+ 7,
+ InputChanges(changed_parts=("text", "needs_you"), changed_report_count=1, max_score_move=0.0),
+ ),
+ ]
+ )
+ def test_names_the_changed_parts(
+ self, _name: str, overrides: dict[str, Any], summary_version: int, expected: InputChanges
+ ) -> None:
+ stored = PersonalInboxRanking(
+ input_parts=input_parts([_candidate(1, _IDS[0])], activity_summary_version=7, focus=None)
+ )
+ current = input_parts(
+ [_candidate(1, _IDS[0], **overrides)], activity_summary_version=summary_version, focus=None
+ )
+
+ assert compare_inputs(stored, current, rank_top=10) == dataclasses.replace(
+ expected, joined_count=0, left_count=0, left_top_count=0
+ )
+
+ def test_counts_the_reports_that_joined_and_left_the_stored_top(self) -> None:
+ ids = [f"018f3d0e-0000-7000-8000-00000000000{index}" for index in range(1, 5)]
+ stored = PersonalInboxRanking(
+ input_parts=input_parts(
+ [_candidate(index, ids[index - 1]) for index in (1, 2, 3)], activity_summary_version=7, focus=None
+ ),
+ result={"ranked": [{"report_id": ids[1]}, {"report_id": ids[0]}, {"report_id": ids[2]}]},
+ )
+ current = input_parts([_candidate(1, ids[0]), _candidate(4, ids[3])], activity_summary_version=7, focus=None)
+
+ changes = compare_inputs(stored, current, rank_top=2)
+
+ assert (changes.changed_parts, changes.joined_count, changes.left_count, changes.left_top_count) == (
+ ("candidate_set",),
+ 1,
+ 2,
+ 1,
+ )
+
+ @parameterized.expand(
+ [
+ ("no_previous_list", None, ("no_previous_list",)),
+ ("row_from_before_the_parts", PersonalInboxRanking(input_parts=None), ("unknown",)),
+ ("unreadable_parts", PersonalInboxRanking(input_parts={"parts": []}), ("unknown",)),
+ ]
+ )
+ def test_a_previous_list_without_parts_reports_why(
+ self, _name: str, previous: PersonalInboxRanking | None, expected: tuple[str, ...]
+ ) -> None:
+ current = input_parts([_candidate(1, _IDS[0])], activity_summary_version=7, focus=None)
+
+ assert compare_inputs(previous, current, rank_top=10) == InputChanges(changed_parts=expected)
+
+
class TestPrompt(SimpleTestCase):
def test_candidates_are_labelled_by_ref_and_the_profile_is_capped(self) -> None:
ids = ["018f3d0e-0000-7000-8000-000000000001", "018f3d0e-0000-7000-8000-000000000002"]
diff --git a/products/signals/backend/test/test_personal_ranking_tick.py b/products/signals/backend/test/test_personal_ranking_tick.py
index f873d59e47cf..1e8e2362fb5c 100644
--- a/products/signals/backend/test/test_personal_ranking_tick.py
+++ b/products/signals/backend/test/test_personal_ranking_tick.py
@@ -264,7 +264,11 @@ class TestTickPair(_TickTest):
self, _name: str, malformed_answers: int, first: str, second: str, model_calls: int
) -> None:
self.malformed_answers = malformed_answers
- assert self._run() == PersonalRankingPairResult(status=first, model_called=True) # type: ignore[arg-type]
+ assert self._run() == PersonalRankingPairResult(
+ status=first, # type: ignore[arg-type]
+ model_called=True,
+ changed_parts=["no_previous_list"],
+ )
assert self.capture.call_args.kwargs["properties"]["trigger"] == "tick"
self._rows().update(created_at=timezone.now() - datetime.timedelta(hours=2))
assert len(select_pairs(self._config(), timezone.now(), 10, [self.team.id]).pairs) == 1
@@ -273,6 +277,9 @@ class TestTickPair(_TickTest):
status=second, # type: ignore[arg-type]
model_called=second == "built",
)
+ if second == "built":
+ # The retried build has the inputs of the failed one, so it reads the stored parts and finds no change.
+ assert self.capture.call_args.kwargs["properties"]["changed_parts"] == []
assert self.create.call_count == model_calls
assert select_pairs(self._config(), timezone.now(), 10, [self.team.id]).pairs == []
@@ -332,6 +339,7 @@ async def _run_tick(
return PersonalRankingPairResult(
status=pair_status, # type: ignore[arg-type]
model_called=pair_status in ("built", "failed"),
+ changed_parts=["scores", "needs_you"] if pair_status == "built" else [],
)
@activity.defn(name="capture_personal_ranking_tick_activity")
@@ -381,6 +389,9 @@ async def test_the_tick_bounds_concurrency_and_stops_at_each_limit(
stop_reason,
)
assert (result.due_pairs, result.enrolled_teams) == (plan.due_pairs, plan.enrolled_teams)
+ assert result.changed_parts == (
+ {"scores": picked, "needs_you": picked} if pair_status == "built" and picked else {}
+ )
assert captured == [result]