Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): drop needs-you claims the facts do not support (#129634)

posthog[bot]410a3cb2ed65
diff --git a/products/signals/backend/personal_ranking/logic.py b/products/signals/backend/personal_ranking/logic.py
index 0378fd8c987d..23105ce9776c 100644
--- a/products/signals/backend/personal_ranking/logic.py
+++ b/products/signals/backend/personal_ranking/logic.py
@@ -562,6 +562,7 @@ def build_personal_ranking(
call.content,
refs=refs,
model_reasons=model_reasons,
+ needs_you={candidate.ref: candidate.needs_you for candidate in candidates},
limit=limit,
rank_top=config.rank_top,
top_max=config.top_max,
diff --git a/products/signals/backend/personal_ranking/prompt.py b/products/signals/backend/personal_ranking/prompt.py
index 9a8d88a8036c..044d9af5ba0e 100644
--- a/products/signals/backend/personal_ranking/prompt.py
+++ b/products/signals/backend/personal_ranking/prompt.py
@@ -12,7 +12,7 @@ from products.signals.backend.activity_summary.read import ActivitySummaryView
from products.signals.backend.personal_ranking.candidates import Candidate, NeedsYouKind
from products.signals.backend.personal_ranking.config import PersonalRankingConfig
-PROMPT_VERSION = 6
+PROMPT_VERSION = 7
Tier = Literal["top", "worth_a_look", "later"]
TIERS: tuple[Tier, ...] = get_args(Tier)
@@ -103,7 +103,8 @@ Each outcome has a probability and a lift. Lift is the probability over the base
Read every candidate. Rank only the best {config.rank_top}, fold the duplicates, and ignore the rest. The candidates you do not rank follow in their given order.
To pick and order the best {config.rank_top}:
-- Put first the reports that need this person: waiting on their input, claimed by them, or naming them as suggested reviewer.
+- Put first the reports that need this person: the ones with a `needs you` line (waiting on their input, claimed by them, or naming them as suggested reviewer).
+- The status in a report's first line, such as `pending_input`, is the report's own state, not a fact about this person. A report with no `needs you` line does not need them, whatever its status.
- Then the reports that match their themes, the objects they own or watch, and the products they work in.
- Use the ranking model's lift as evidence of value. Prefer lift to the raw probability. A report with an open pull request is cheap to finish.
- Old reports with no activity and reports far from their work go last.
@@ -112,8 +113,8 @@ To pick and order the best {config.rank_top}:
Output:
- `ranked`: at most {config.rank_top} entries, the best candidates that you do not fold, with `ref` (such as "c3"), `rank` (1 is first, no gaps, no repeats) and `tier`.
- `tier`: "top" for at most {config.top_max} reports they should look at today. A "top" report needs a needs-you fact, a match to one of their themes, an object they own, or the focus. "worth_a_look" for at most {config.worth_a_look_max} other useful reports. "later" for the rest. A lower rank never has a better tier.
-- `reason`: why this rank, in plain words. Name the theme, object, product or needs-you fact that matched. Never quote the profile. At most {reasons.top} characters for "top", {reasons.worth_a_look} for "worth_a_look", {reasons.later} for "later". Every entry needs a reason.
-- `matched`: the theme label that matched (as written in the person block) and the needs-you kinds that apply, or null when nothing matched.
+- `reason`: why this rank, in plain words. Name the theme, object, product or needs-you fact that matched. Name a needs-you fact only when the report's `needs you` line states it. Never quote the profile. At most {reasons.top} characters for "top", {reasons.worth_a_look} for "worth_a_look", {reasons.later} for "later". Every entry needs a reason.
+- `matched`: the theme label that matched (as written in the person block) and the needs-you kinds from the report's `needs you` line (empty when it has none), or null when nothing matched.
- `next_step`: one short action for ranks 1 to {min(limit, config.rank_top)}, at most {config.next_step_max_chars} characters, else null.
- `folded`: any candidate, ranked or not, that repeats another candidate (a digest series or a duplicate) goes here, not in `ranked`, with `into` (the ref it repeats) and a short reason.
diff --git a/products/signals/backend/personal_ranking/validation.py b/products/signals/backend/personal_ranking/validation.py
index 77b8e5b734b7..d8adcd9a3199 100644
--- a/products/signals/backend/personal_ranking/validation.py
+++ b/products/signals/backend/personal_ranking/validation.py
@@ -63,6 +63,8 @@ class ValidationCounts:
tier_demotions: int = 0
# Folds of an unknown ref, into an unknown or folded ref, or into the ref itself.
invalid_folds: int = 0
+ # Ranked entries that claim a needs-you kind the facts do not hold. The kind drops and the reason is replaced.
+ false_needs_you: int = 0
@frozen
@@ -168,6 +170,10 @@ def _folds(items: list[Any], known: set[str], limits: ReasonLimits) -> tuple[dic
return valid, invalid + len(proposed) - len(valid)
+def _theme_reason(theme: str) -> str:
+ return f"Matches your recent work: {theme}."
+
+
def model_order(refs: Sequence[str], model_reasons: Mapping[str, str], limit: int) -> tuple[RankedEntry, ...]:
"""The fallback list: every candidate in model order, the first `limit` as worth a look."""
return tuple(
@@ -189,6 +195,7 @@ def validate_ranking(
*,
refs: Sequence[str],
model_reasons: Mapping[str, str],
+ needs_you: Mapping[str, Sequence[NeedsYouKind]],
limit: int,
rank_top: int,
top_max: int,
@@ -196,7 +203,10 @@ def validate_ranking(
reason_limits: ReasonLimits,
next_step_max_chars: int,
) -> ValidatedRanking:
- """`refs` are the candidates in model order. Raises `MalformedRankingError` when the answer cannot be used at all."""
+ """`refs` are the candidates in model order, and `needs_you` holds each one's real needs-you facts.
+
+ Raises `MalformedRankingError` when the answer cannot be used at all.
+ """
raw, folded_items = _parse(content)
known = set(refs)
folds, invalid_folds = _folds(folded_items, known, reason_limits)
@@ -220,7 +230,7 @@ def validate_ranking(
caps: dict[Tier, int] = {"top": top_max, "worth_a_look": worth_a_look_max}
tier_sizes: Counter[Tier] = Counter()
entries: list[RankedEntry] = []
- renumbered = tier_corrections = tier_demotions = 0
+ renumbered = tier_corrections = tier_demotions = false_needs_you = 0
# Corrections follow the model's own tiers, so a demotion is not counted again as a correction.
worst_corrected = worst_tier = 0
for rank, entry in enumerate(top, start=1):
@@ -236,6 +246,13 @@ def validate_ranking(
tier = TIERS[tier_index]
tier_sizes[tier] += 1
reason = entry.reason.strip() or model_reasons[entry.ref]
+ matched = entry.matched
+ facts = needs_you.get(entry.ref, ())
+ if matched is not None and any(kind not in facts for kind in matched.needs_you):
+ false_needs_you += 1
+ matched = Matched(theme=matched.theme, needs_you=tuple(kind for kind in matched.needs_you if kind in facts))
+ # The model's reason may repeat the false claim. A real needs-you fact outranks the theme.
+ reason = _theme_reason(matched.theme) if matched.theme and not facts else model_reasons[entry.ref]
entries.append(
RankedEntry(
ref=entry.ref,
@@ -243,7 +260,7 @@ def validate_ranking(
tier=tier,
reason=cap(reason, _reason_limit(tier, reason_limits)),
next_step=cap(entry.next_step, next_step_max_chars) if entry.next_step and rank <= limit else None,
- matched=entry.matched,
+ matched=matched,
ranked_by="llm",
)
)
@@ -273,5 +290,6 @@ def validate_ranking(
tier_corrections=tier_corrections,
tier_demotions=tier_demotions,
invalid_folds=invalid_folds,
+ false_needs_you=false_needs_you,
),
)
diff --git a/products/signals/backend/test/test_personal_ranking.py b/products/signals/backend/test/test_personal_ranking.py
index e65ec88304f8..b477aff56c9c 100644
--- a/products/signals/backend/test/test_personal_ranking.py
+++ b/products/signals/backend/test/test_personal_ranking.py
@@ -62,19 +62,34 @@ MODEL_REASONS = {ref: f"model {ref}" for ref in REFS}
LIMITS = ReasonLimits(top=40, worth_a_look=30, later=20)
-def _entry(ref: str, rank: Any, tier: str = "worth_a_look", reason: str = "matches", next_step: str | None = None):
- return {"ref": ref, "rank": rank, "tier": tier, "reason": reason, "matched": None, "next_step": next_step}
+def _entry(
+ ref: str,
+ rank: Any,
+ tier: str = "worth_a_look",
+ reason: str = "matches",
+ next_step: str | None = None,
+ matched: dict | None = None,
+):
+ return {"ref": ref, "rank": rank, "tier": tier, "reason": reason, "matched": matched, "next_step": next_step}
def _answer(ranked: list[dict], folded: list[dict] | None = None) -> str:
return json.dumps({"ranked": ranked, "folded": folded or []})
-def _validate(content: str | None, limit: int = 2, rank_top: int = 10, top_max: int = 5, worth_a_look_max: int = 10):
+def _validate(
+ content: str | None,
+ limit: int = 2,
+ rank_top: int = 10,
+ top_max: int = 5,
+ worth_a_look_max: int = 10,
+ needs_you: dict[str, tuple] | None = None,
+):
return validate_ranking(
content,
refs=REFS,
model_reasons=MODEL_REASONS,
+ needs_you=needs_you or {},
limit=limit,
rank_top=rank_top,
top_max=top_max,
@@ -229,6 +244,62 @@ class TestValidateRanking(SimpleTestCase):
assert result.counts.tier_demotions == expected_demotions
assert result.counts.tier_corrections == 0
+ @parameterized.expand(
+ [
+ (
+ "status_read_as_a_personal_claim_falls_back_to_the_theme",
+ {"theme": "Checkout", "needs_you": ["waiting_on_input"]},
+ (),
+ {"theme": "Checkout", "needs_you": ()},
+ "Matches your recent work: Checkout.",
+ 1,
+ ),
+ (
+ "false_claim_with_no_theme_gets_the_model_reason",
+ {"theme": None, "needs_you": ["suggested_reviewer"]},
+ (),
+ {"theme": None, "needs_you": ()},
+ "model c1",
+ 1,
+ ),
+ (
+ "only_the_false_kind_drops_and_the_real_fact_wins_over_the_theme",
+ {"theme": "Checkout", "needs_you": ["waiting_on_input", "claimed"]},
+ ("claimed",),
+ {"theme": "Checkout", "needs_you": ("claimed",)},
+ "model c1",
+ 1,
+ ),
+ (
+ "true_claim_keeps_the_model_reason",
+ {"theme": None, "needs_you": ["claimed"]},
+ ("claimed",),
+ {"theme": None, "needs_you": ("claimed",)},
+ "Waiting on you",
+ 0,
+ ),
+ ]
+ )
+ def test_needs_you_claims_must_match_the_facts(
+ self,
+ _name: str,
+ matched: dict,
+ facts: tuple,
+ expected_matched: dict,
+ expected_reason: str,
+ expected_false: int,
+ ) -> None:
+ result = _validate(
+ _answer([_entry("c1", 1, tier="top", reason="Waiting on you", matched=matched)]),
+ needs_you={"c1": facts},
+ )
+
+ entry = result.entries[0]
+ assert (entry.rank, entry.tier) == (1, "top")
+ assert dataclasses.asdict(entry.matched) == expected_matched
+ assert entry.reason == expected_reason
+ assert result.counts.false_needs_you == expected_false
+
@parameterized.expand(
[
("empty", None),
@@ -287,7 +358,7 @@ class TestInputParts(SimpleTestCase):
assert (
input_hash(candidates, activity_summary_version=7, focus="checkout")
- == "8b3bc5b28875163b6769d657f08d4351f846fbd0293ca46563bd930743ed5958"
+ == "f942040677219aed0d042d2e28ac23bb82ae2e26e40fe21eb10d7d2b17466fdd"
)
@parameterized.expand(