:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): record which inputs changed when a personal ranking rebuilds (#129353)
posthog[bot]d64403bf2a76
diff --git a/products/signals/backend/migrations/0233_personalinboxranking_input_parts.py b/products/signals/backend/migrations/0233_personalinboxranking_input_parts.pynew file mode 100644index 000000000000..04719655616c--- /dev/null+++ b/products/signals/backend/migrations/0233_personalinboxranking_input_parts.py@@ -0,0 +1,13 @@+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [("signals", "0232_signalreport_created_by_index")]++ operations = [+ migrations.AddField(+ model_name="personalinboxranking",+ name="input_parts",+ field=models.JSONField(blank=True, null=True),+ ),+ ]diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txtindex 5ce15909f164..ae6c210e7273 100644--- a/products/signals/backend/migrations/max_migration.txt+++ b/products/signals/backend/migrations/max_migration.txt@@ -1 +1 @@-0232_signalreport_created_by_index+0233_personalinboxranking_input_partsdiff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex 79ea8a9bba1e..5461576566e3 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -4372,6 +4372,8 @@ class PersonalInboxRanking(TeamScopedRootMixin, UUIDModel):# Covers the candidates with their status, priority, PR state, needs-you facts and score, the summary version, the prompt version and the focus.input_hash = models.CharField(max_length=64)+ # The inputs of `input_hash` as one hash per part, from `input_parts`. Null on rows from before the parts.+ input_parts = models.JSONField(null=True, blank=True)# Report ids in candidate order.candidate_ids = models.JSONField(default=list)# Every candidate with its rank, tier and reason, and the folded reports. See `PersonalRankingResult.as_json`.diff --git a/products/signals/backend/personal_ranking/logic.py b/products/signals/backend/personal_ranking/logic.pyindex 9ea94dcc8fc7..0378fd8c987d 100644--- a/products/signals/backend/personal_ranking/logic.py+++ b/products/signals/backend/personal_ranking/logic.py@@ -71,6 +71,15 @@ RETRYABLE_FALLBACK_REASONS = frozenset({FALLBACK_LLM_ERROR, FALLBACK_TIMEOUT, FALLBACK_MALFORMED, FALLBACK_TRUNCATED, FALLBACK_AI_NOT_APPROVED})+# The input parts that `input_parts` hashes one by one. The report parts have one hash for each candidate.+REPORT_PARTS = ("text", "status", "priority", "pull_request_state", "needs_you", "scores")+INPUT_PARTS = ("candidate_set", *REPORT_PARTS, "activity_summary_version", "prompt_version", "focus")+# The person had no list before this build.+CHANGED_NO_PREVIOUS = "no_previous_list"+# The previous list has no part hashes, so the build cannot tell which part changed.+CHANGED_UNKNOWN = "unknown"+_PART_HASH_CHARS = 16+BuildStatus = Literal["ok", "fallback", "empty", "disabled"]# What started a build, on the build event and the trace, so cost can be split by it.BuildTrigger = Literal["on_demand", "tick"]@@ -92,6 +101,32 @@ class _ModelCall:fallback_reason: str | None+@frozen+class InputChanges:+ """How the inputs of a build differ from the inputs of the person's previous list."""++ # Names from `INPUT_PARTS`, or only `CHANGED_NO_PREVIOUS` or `CHANGED_UNKNOWN`.+ changed_parts: tuple[str, ...]+ # The reports in both lists with a changed report part. Null when the parts are not comparable.+ changed_report_count: int | None = None+ # The largest probability move of a report in both lists. Null when no probability is in both lists.+ max_score_move: float | None = None+ joined_count: int | None = None+ left_count: int | None = None+ # The reports that left and were in the first `rank_top` entries of the previous list.+ left_top_count: int | None = None++ def properties(self) -> dict[str, Any]:+ return {+ "changed_parts": list(self.changed_parts),+ "changed_report_count": self.changed_report_count,+ "max_score_move": self.max_score_move,+ "candidates_joined": self.joined_count,+ "candidates_left": self.left_count,+ "candidates_left_top": self.left_top_count,+ }++@frozenclass PersonalRankingBuild:status: BuildStatus@@ -111,6 +146,10 @@ class PersonalRankingBuild:latency_ms: int | None = None# None for a dry run or a disabled build, which store nothing.ranking_id: str | None = None+ # The stored `input_parts`. None for a disabled build.+ input_parts: dict[str, Any] | None = None+ # None for a dry run or a disabled build.+ input_changes: InputChanges | None = Nonedef result_json(self) -> dict[str, Any]:"""The stored `result`: every candidate by report id, with its rank, tier and reason."""@@ -187,6 +226,104 @@ def input_hash(candidates: list[Candidate], *, activity_summary_version: int | Nreturn hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode()).hexdigest()+def _part_hash(value: Any) -> str:+ return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode()).hexdigest()[:_PART_HASH_CHARS]+++def _report_parts(candidate: Candidate) -> dict[str, Any]:+ predictions = candidate.predictions+ return {+ "text": [candidate.title, candidate.summary_lead],+ "status": candidate.status,+ "priority": candidate.priority,+ "pull_request_state": candidate.pull_request_state,+ "needs_you": list(candidate.needs_you),+ "scores": [+ predictions.status if predictions else None,+ [[entry.outcome, round(entry.probability, 2)] for entry in predictions.predictions] if predictions else [],+ ],+ }+++def input_parts(+ candidates: list[Candidate], *, activity_summary_version: int | None, focus: str | None+) -> dict[str, Any]:+ """The inputs of `input_hash` as one hash per part, so a later build can name the parts that changed.++ The report parts have one hash for each report. `scores` keeps the probabilities, so a later build can measure the move.+ """+ return {+ "parts": {+ "activity_summary_version": _part_hash(activity_summary_version),+ "prompt_version": _part_hash(PROMPT_VERSION),+ "focus": _part_hash(focus),+ },+ "reports": {+ candidate.report_id: {part: _part_hash(value) for part, value in _report_parts(candidate).items()}+ for candidate in candidates+ },+ "scores": {+ candidate.report_id: {+ entry.outcome: round(entry.probability, 4) for entry in candidate.predictions.predictions+ }+ for candidate in candidates+ if candidate.predictions+ },+ }+++def _max_score_move(previous: dict[str, dict[str, float]], current: dict[str, dict[str, float]]) -> float | None:+ moves = [+ abs(probability - previous[report_id][outcome])+ for report_id, scores in current.items()+ if report_id in previous+ for outcome, probability in scores.items()+ if outcome in previous[report_id]+ ]+ return round(max(moves), 4) if moves else None+++def compare_inputs(previous: PersonalInboxRanking | None, current: dict[str, Any], *, rank_top: int) -> InputChanges:+ """The parts that differ between the previous list and the current inputs. Never raises."""+ if previous is None:+ return InputChanges(changed_parts=(CHANGED_NO_PREVIOUS,))+ stored = previous.input_parts+ try:+ if not stored:+ return InputChanges(changed_parts=(CHANGED_UNKNOWN,))+ stored_reports: dict[str, dict[str, str]] = stored["reports"]+ current_reports: dict[str, dict[str, str]] = current["reports"]+ joined = current_reports.keys() - stored_reports.keys()+ left = stored_reports.keys() - current_reports.keys()+ changed = {"candidate_set"} if joined or left else set()+ changed |= {part for part in current["parts"] if current["parts"][part] != stored["parts"].get(part)}+ changed_reports = 0+ for report_id in current_reports.keys() & stored_reports.keys():+ report_changed = {+ part+ for part in REPORT_PARTS+ if current_reports[report_id].get(part) != stored_reports[report_id].get(part)+ }+ changed |= report_changed+ changed_reports += bool(report_changed)+ top_ids = {entry["report_id"] for entry in previous.result.get("ranked", [])[:rank_top]}+ return InputChanges(+ changed_parts=tuple(part for part in INPUT_PARTS if part in changed),+ changed_report_count=changed_reports,+ max_score_move=_max_score_move(stored.get("scores", {}), current["scores"]),+ joined_count=len(joined),+ left_count=len(left),+ left_top_count=len(left & top_ids),+ )+ except (KeyError, TypeError, AttributeError):+ logger.warning("signals.personal_ranking.unreadable_input_parts", ranking_id=str(previous.id))+ return InputChanges(changed_parts=(CHANGED_UNKNOWN,))+++def latest_row(team: Team, user: User) -> PersonalInboxRanking | None:+ return PersonalInboxRanking.objects.for_team(team.id).filter(user_id=user.id).order_by("-created_at", "-id").first()++def _client(team: Team, user: User, now: datetime.datetime, config: PersonalRankingConfig, trigger: BuildTrigger) -> OpenAI:@@ -317,6 +454,7 @@ def _capture("tokens_in": build.tokens_in,"tokens_out": build.tokens_out,"latency_ms": build.latency_ms,+ **(build.input_changes.properties() if build.input_changes is not None else {}),},groups={"organization": str(team.organization_id), "project": str(team.uuid)},)@@ -330,6 +468,7 @@ def _store(team: Team, user: User, build: PersonalRankingBuild) -> PersonalRankiteam_id=team.id,user_id=user.id,input_hash=build.input_hash or "",+ input_parts=build.input_parts,candidate_ids=[candidate.report_id for candidate in build.candidates],result=build.result_json(),personalized=build.personalized,@@ -437,6 +576,7 @@ def build_personal_ranking(fallback_reason = FALLBACK_MALFORMEDstatus = "ok" if validated is not None else "fallback"+ parts = input_parts(candidates, activity_summary_version=summary_version, focus=focus)build = PersonalRankingBuild(status=status,personalized=validated is not None,@@ -449,6 +589,7 @@ def build_personal_ranking(counts=validated.counts if validated is not None else ValidationCounts(),limit=limit,input_hash=input_hash(candidates, activity_summary_version=summary_version, focus=focus),+ input_parts=parts,activity_summary_version=summary_version,model=config.model,model_called=call is not None,@@ -458,7 +599,8 @@ def build_personal_ranking()if dry_run:return build- build = _store(team, user, build)+ changes = compare_inputs(latest_row(team, user), parts, rank_top=config.rank_top)+ build = _store(team, user, dataclasses.replace(build, input_changes=changes))try:_prune(team, user, config.max_rows_per_person)except Exception:diff --git a/products/signals/backend/personal_ranking/serve.py b/products/signals/backend/personal_ranking/serve.pyindex 6ae4b7935646..76ea0557dc56 100644--- a/products/signals/backend/personal_ranking/serve.py+++ b/products/signals/backend/personal_ranking/serve.py@@ -38,6 +38,7 @@ from products.signals.backend.personal_ranking.candidates import Candidate, gathfrom products.signals.backend.personal_ranking.config import PersonalRankingConfig, read_configfrom products.signals.backend.personal_ranking.logic import (input_hash,+ latest_row,may_call_model,model_order_reason,normalize_focus,@@ -216,10 +217,6 @@ def _start_build(return "failed"-def latest_row(team: Team, user: User) -> PersonalInboxRanking | None:- return PersonalInboxRanking.objects.for_team(team.id).filter(user_id=user.id).order_by("-created_at", "-id").first()--def _row_stored_since(team: Team, user: User, seen: PersonalInboxRanking | None, current_hash: str) -> PersonalInboxRanking | None:diff --git a/products/signals/backend/personal_ranking/tick.py b/products/signals/backend/personal_ranking/tick.pyindex e5872660a8b9..8268114ec26a 100644--- a/products/signals/backend/personal_ranking/tick.py+++ b/products/signals/backend/personal_ranking/tick.py@@ -50,13 +50,17 @@ from products.signals.backend.personal_ranking.config import (read_config,)from products.signals.backend.personal_ranking.logic import (+ CHANGED_NO_PREVIOUS,+ CHANGED_UNKNOWN,+ INPUT_PARTS,PIPELINE_DISTINCT_ID,RETRYABLE_FALLBACK_REASONS,build_personal_ranking,input_hash,+ latest_row,may_call_model,)-from products.signals.backend.personal_ranking.serve import latest_row, ready_summary_version+from products.signals.backend.personal_ranking.serve import ready_summary_versionfrom products.signals.backend.personal_ranking_tasks import BUILD_LOCK_TTL_SECONDS, build_lock_keylogger = structlog.get_logger(__name__)@@ -160,6 +164,8 @@ class PersonalRankingTickPlan:class PersonalRankingPairResult:status: PairStatusmodel_called: bool = False+ # The `changed_parts` of the build. Empty when the pair did not build.+ changed_parts: list[str] = dataclasses.field(default_factory=list)@frozen@@ -169,6 +175,8 @@ class PersonalRankingTickResult:due_without_list: int = 0picked: int = 0statuses: dict[str, int] = dataclasses.field(default_factory=dict)+ # Built pairs by changed input part. A pair with two changed parts counts in both.+ changed_parts: dict[str, int] = dataclasses.field(default_factory=dict)errored: int = 0stop_reason: str | None = Noneoldest_due_hours: float | None = None@@ -373,7 +381,11 @@ def _check_and_build(team: Team, user: User, config: PersonalRankingConfig) -> Pif build.status == "disabled":return PersonalRankingPairResult(status="disabled")failed = build.model_called and build.status == "fallback"- return PersonalRankingPairResult(status="failed" if failed else "built", model_called=build.model_called)+ return PersonalRankingPairResult(+ status="failed" if failed else "built",+ model_called=build.model_called,+ changed_parts=list(build.input_changes.changed_parts) if build.input_changes is not None else [],+ )def run_pair(pair: TickPair) -> PersonalRankingPairResult:@@ -399,6 +411,10 @@ def capture_tick(result: PersonalRankingTickResult) -> None:"due_without_list": result.due_without_list,"picked": result.picked,**{status: result.statuses.get(status, 0) for status in PAIR_STATUSES},+ **{+ f"changed_{part}": result.changed_parts.get(part, 0)+ for part in (*INPUT_PARTS, CHANGED_NO_PREVIOUS, CHANGED_UNKNOWN)+ },"errored": result.errored,"stop_reason": result.stop_reason,"oldest_due_hours": result.oldest_due_hours,@@ -450,6 +466,7 @@ class PersonalRankingTickWorkflow:deadline = started + datetime.timedelta(seconds=plan.time_budget_seconds)statuses: dict[str, int] = {}+ changed_parts: dict[str, int] = {}errored = 0# Model calls of this tick and the calls among them that failed or raised, for the circuit breaker.calls = 0@@ -491,6 +508,8 @@ class PersonalRankingTickWorkflow:failures += 1continuestatuses[outcome.status] = statuses.get(outcome.status, 0) + 1+ for part in outcome.changed_parts:+ changed_parts[part] = changed_parts.get(part, 0) + 1calls += outcome.model_calledfailures += outcome.status == "failed"limit_stop = limit_stop or _LIMIT_STOPS.get(outcome.status)@@ -504,6 +523,7 @@ class PersonalRankingTickWorkflow:due_without_list=plan.due_without_list,picked=picked,statuses=statuses,+ changed_parts=changed_parts,errored=errored,stop_reason=stop_reason,oldest_due_hours=plan.oldest_due_hours,diff --git a/products/signals/backend/test/test_personal_ranking.py b/products/signals/backend/test/test_personal_ranking.pyindex d2e8a1fd5d35..e65ec88304f8 100644--- a/products/signals/backend/test/test_personal_ranking.py+++ b/products/signals/backend/test/test_personal_ranking.py@@ -44,7 +44,14 @@ from products.signals.backend.personal_ranking.config import (ReasonLimits,parse_config,)-from products.signals.backend.personal_ranking.logic import BUILT_EVENT, build_personal_ranking+from products.signals.backend.personal_ranking.logic import (+ BUILT_EVENT,+ InputChanges,+ build_personal_ranking,+ compare_inputs,+ input_hash,+ input_parts,+)from products.signals.backend.personal_ranking.prompt import render_person, system_prompt, user_messagefrom products.signals.backend.personal_ranking.validation import MalformedRankingError, validate_rankingfrom products.signals.backend.personal_ranking_tasks import build_lock_key, build_personal_inbox_ranking@@ -263,6 +270,103 @@ def _candidate(index: int, report_id: str, **overrides: Any) -> Candidate:return Candidate(**{**fields, **overrides})+_IDS = ["018f3d0e-0000-7000-8000-000000000001", "018f3d0e-0000-7000-8000-000000000002"]+++def _scored(probability: float) -> ReportPredictions:+ return ReportPredictions(+ status="scored",+ model="m@1",+ predictions=(OutcomePrediction(outcome="action", probability=probability, lift=None, readable=True),),+ )+++class TestInputParts(SimpleTestCase):+ def test_input_hash_keeps_its_value(self) -> None:+ candidates = [_candidate(1, _IDS[0]), _candidate(2, _IDS[1], predictions=None, needs_you=())]++ assert (+ input_hash(candidates, activity_summary_version=7, focus="checkout")+ == "8b3bc5b28875163b6769d657f08d4351f846fbd0293ca46563bd930743ed5958"+ )++ @parameterized.expand(+ [+ ("nothing", {}, 7, InputChanges(changed_parts=(), changed_report_count=0, max_score_move=0.0)),+ (+ "score_jitter_under_the_rounding",+ {"predictions": _scored(0.421)},+ 7,+ InputChanges(changed_parts=(), changed_report_count=0, max_score_move=0.001),+ ),+ (+ "score",+ {"predictions": _scored(0.5)},+ 7,+ InputChanges(changed_parts=("scores",), changed_report_count=1, max_score_move=0.08),+ ),+ (+ "summary_version",+ {},+ 8,+ InputChanges(changed_parts=("activity_summary_version",), changed_report_count=0, max_score_move=0.0),+ ),+ (+ "text_and_needs_you",+ {"title": "Renamed", "needs_you": ("claimed",)},+ 7,+ InputChanges(changed_parts=("text", "needs_you"), changed_report_count=1, max_score_move=0.0),+ ),+ ]+ )+ def test_names_the_changed_parts(+ self, _name: str, overrides: dict[str, Any], summary_version: int, expected: InputChanges+ ) -> None:+ stored = PersonalInboxRanking(+ input_parts=input_parts([_candidate(1, _IDS[0])], activity_summary_version=7, focus=None)+ )+ current = input_parts(+ [_candidate(1, _IDS[0], **overrides)], activity_summary_version=summary_version, focus=None+ )++ assert compare_inputs(stored, current, rank_top=10) == dataclasses.replace(+ expected, joined_count=0, left_count=0, left_top_count=0+ )++ def test_counts_the_reports_that_joined_and_left_the_stored_top(self) -> None:+ ids = [f"018f3d0e-0000-7000-8000-00000000000{index}" for index in range(1, 5)]+ stored = PersonalInboxRanking(+ input_parts=input_parts(+ [_candidate(index, ids[index - 1]) for index in (1, 2, 3)], activity_summary_version=7, focus=None+ ),+ result={"ranked": [{"report_id": ids[1]}, {"report_id": ids[0]}, {"report_id": ids[2]}]},+ )+ current = input_parts([_candidate(1, ids[0]), _candidate(4, ids[3])], activity_summary_version=7, focus=None)++ changes = compare_inputs(stored, current, rank_top=2)++ assert (changes.changed_parts, changes.joined_count, changes.left_count, changes.left_top_count) == (+ ("candidate_set",),+ 1,+ 2,+ 1,+ )++ @parameterized.expand(+ [+ ("no_previous_list", None, ("no_previous_list",)),+ ("row_from_before_the_parts", PersonalInboxRanking(input_parts=None), ("unknown",)),+ ("unreadable_parts", PersonalInboxRanking(input_parts={"parts": []}), ("unknown",)),+ ]+ )+ def test_a_previous_list_without_parts_reports_why(+ self, _name: str, previous: PersonalInboxRanking | None, expected: tuple[str, ...]+ ) -> None:+ current = input_parts([_candidate(1, _IDS[0])], activity_summary_version=7, focus=None)++ assert compare_inputs(previous, current, rank_top=10) == InputChanges(changed_parts=expected)++class TestPrompt(SimpleTestCase):def test_candidates_are_labelled_by_ref_and_the_profile_is_capped(self) -> None:ids = ["018f3d0e-0000-7000-8000-000000000001", "018f3d0e-0000-7000-8000-000000000002"]diff --git a/products/signals/backend/test/test_personal_ranking_tick.py b/products/signals/backend/test/test_personal_ranking_tick.pyindex f873d59e47cf..1e8e2362fb5c 100644--- a/products/signals/backend/test/test_personal_ranking_tick.py+++ b/products/signals/backend/test/test_personal_ranking_tick.py@@ -264,7 +264,11 @@ class TestTickPair(_TickTest):self, _name: str, malformed_answers: int, first: str, second: str, model_calls: int) -> None:self.malformed_answers = malformed_answers- assert self._run() == PersonalRankingPairResult(status=first, model_called=True) # type: ignore[arg-type]+ assert self._run() == PersonalRankingPairResult(+ status=first, # type: ignore[arg-type]+ model_called=True,+ changed_parts=["no_previous_list"],+ )assert self.capture.call_args.kwargs["properties"]["trigger"] == "tick"self._rows().update(created_at=timezone.now() - datetime.timedelta(hours=2))assert len(select_pairs(self._config(), timezone.now(), 10, [self.team.id]).pairs) == 1@@ -273,6 +277,9 @@ class TestTickPair(_TickTest):status=second, # type: ignore[arg-type]model_called=second == "built",)+ if second == "built":+ # The retried build has the inputs of the failed one, so it reads the stored parts and finds no change.+ assert self.capture.call_args.kwargs["properties"]["changed_parts"] == []assert self.create.call_count == model_callsassert select_pairs(self._config(), timezone.now(), 10, [self.team.id]).pairs == []@@ -332,6 +339,7 @@ async def _run_tick(return PersonalRankingPairResult(status=pair_status, # type: ignore[arg-type]model_called=pair_status in ("built", "failed"),+ changed_parts=["scores", "needs_you"] if pair_status == "built" else [],)@activity.defn(name="capture_personal_ranking_tick_activity")@@ -381,6 +389,9 @@ async def test_the_tick_bounds_concurrency_and_stops_at_each_limit(stop_reason,)assert (result.due_pairs, result.enrolled_teams) == (plan.due_pairs, plan.enrolled_teams)+ assert result.changed_parts == (+ {"scores": picked, "needs_you": picked} if pair_status == "built" and picked else {}+ )assert captured == [result]