Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(replay-vision): move pii check and search rerank to jeeves (#120387)

Rory Shanks4ecd3aa34d46
diff --git a/products/replay_vision/backend/search_rerank.py b/products/replay_vision/backend/search_rerank.py
index 897bddaf75c4..07c0f55f080c 100644
--- a/products/replay_vision/backend/search_rerank.py
+++ b/products/replay_vision/backend/search_rerank.py
@@ -26,7 +26,7 @@ from products.replay_vision.backend.temporal.metrics import record_search_rerank
logger = structlog.get_logger(__name__)
-RERANK_MODEL = "posthog/hogference/jevk5-fp8-0.2"
+RERANK_MODEL = "posthog/hogference/jeeves-0.1"
RERANK_CANDIDATES = 20
RERANK_TEXT_CHARS = 500
RERANK_REQUESTS = 2
diff --git a/products/replay_vision/backend/temporal/pii_check.py b/products/replay_vision/backend/temporal/pii_check.py
index 55cd5b3ff106..e43b3311ba6d 100644
--- a/products/replay_vision/backend/temporal/pii_check.py
+++ b/products/replay_vision/backend/temporal/pii_check.py
@@ -2,7 +2,10 @@
Jev answers two separate questions: whether the scanner's question asks who the session belongs to, and whether
the answer contains personal data. Asked as one question, with the session's identity in view, it could not tell the
-two apart. A flagged answer gets one text-only rewrite, and an answer still flagged after it fails the observation as
+two apart. The personal-data check itself asks about names and about contact details or IDs as two questions in one
+request, because a single question that listed every kind of personal data also flagged company names and business
+contact details.
+A flagged answer gets one text-only rewrite, and an answer still flagged after it fails the observation as
`pii_detected`. Where Jev is unavailable or errors, the answer passes.
"""
@@ -30,25 +33,54 @@ logger = structlog.get_logger(__name__)
_OutputT = TypeVar("_OutputT", bound=BaseModel)
-PII_CHECK_MODEL = "posthog/hogference/jevk5-fp8-0.2"
-# Calibrated offline on production answers: at these values the detector caught 96% of answers that repeated the
-# subject's email unasked, most other flags were real names, and a question that asks for identity scored 0.86.
-PII_FLAG_PROBABILITY = 0.8
+PII_CHECK_MODEL = "posthog/hogference/jeeves-0.1"
+# Calibrated offline in October 2026 on a blind-labelled sample of production answers. Jeeves scores most real names
+# between 0.4 and 0.8, so a higher bar misses most of them. Answers without personal data score near 0.1,
+# which leaves 0.4 well clear of them. Questions that ask for identity separate cleanly at 0.5.
+PII_FLAG_PROBABILITY = 0.4
ASKS_FOR_IDENTITY_PROBABILITY = 0.5
_JEV_TIMEOUT_SECONDS = 10.0
_REWRITE_MODEL = "gemini-3.8-flash"
_TEXT_FIELDS = ("title", "summary", "reasoning", "notability_reason")
-_DETECT_QUESTION = (
- "The state holds the text an AI scanner wrote about one recorded user session. Does the text contain personal data "
- "about a specific person: an email address, a person's name, a username or handle, a phone number, a street or IP "
- "address, payment details, or an account or government ID? Generic references such as 'the user', 'a customer', "
- "or a company or product name are not personal data."
+_NAME_QUESTION = DecisionQuestion(
+ type=DecisionQuestionType.NOUL,
+ instructions=(
+ "The state holds the text an AI scanner wrote about one recorded user session. Does the text name a specific "
+ "real person, such as the user, a customer, a contact, or a staff member?"
+ ),
+ criteria={
+ "true": "The text gives a real individual's first name, full name, or surname.",
+ "false": (
+ "People are referred to generically ('the user', 'a customer'), or the only names are of companies, shops, "
+ "products, pages, AI personas, or test records, even when such a name contains a first name."
+ ),
+ },
)
-_ASKS_QUESTION = (
- "The state holds the question a product team gave an AI scanner that watches recorded user sessions. Does the "
- "question ask the scanner to identify who the session belongs to, such as the person's name, email, or their "
- "company?"
+_CONTACT_QUESTION = DecisionQuestion(
+ type=DecisionQuestionType.NOUL,
+ instructions=(
+ "The state holds the text an AI scanner wrote about one recorded user session. Does the text show contact "
+ "details or an identifier that belongs to a specific person?"
+ ),
+ criteria={
+ "true": (
+ "The text shows an actual personal email address, phone number, street or IP address, username or handle, "
+ "payment details, or an account, record or government ID."
+ ),
+ "false": (
+ "No such value appears, the text only says one was entered, or the value belongs to a business: a company "
+ "inbox, a business phone number, or a bare email domain."
+ ),
+ },
+)
+_ASKS_QUESTION = DecisionQuestion(
+ type=DecisionQuestionType.NOUL,
+ instructions=(
+ "The state holds the question a product team gave an AI scanner that watches recorded user sessions. Does the "
+ "question ask the scanner to identify who the session belongs to, such as the person's name, email, or their "
+ "company?"
+ ),
)
_REWRITE_INSTRUCTIONS = """\
@@ -92,7 +124,9 @@ def answer_text(output: BaseModel) -> dict[str, str]:
}
-def _yes_probability(team_id: int, state: dict[str, Any], instructions: str, trace_id: str) -> float | None:
+def _highest_yes_probability(
+ team_id: int, state: dict[str, Any], questions: dict[str, DecisionQuestion], trace_id: str
+) -> float | None:
if not decision_api.decisions_available_here():
return None
try:
@@ -100,7 +134,7 @@ def _yes_probability(team_id: int, state: dict[str, Any], instructions: str, tra
DecisionRequest(
team_id=team_id,
state=state,
- questions={"q": DecisionQuestion(type=DecisionQuestionType.NOUL, instructions=instructions)},
+ questions=questions,
model=PII_CHECK_MODEL,
ai_product="replay_vision",
trace_id=trace_id,
@@ -112,14 +146,19 @@ def _yes_probability(team_id: int, state: dict[str, Any], instructions: str, tra
except Exception:
logger.warning("replay_vision.pii_check.unavailable", team_id=team_id, exc_info=True)
return None
- return getattr(result.answers.get("q"), "probability", None)
+ probabilities = [
+ probability
+ for question_id in questions
+ if (probability := getattr(result.answers.get(question_id), "probability", None)) is not None
+ ]
+ return max(probabilities) if len(probabilities) == len(questions) else None
def asks_for_identity(*, team_id: int, question: str, trace_id: str) -> bool | None:
"""Whether the scanner's question asks who the session belongs to; None when Jev could not answer."""
if not question.strip():
return False
- probability = _yes_probability(team_id, {"question": question}, _ASKS_QUESTION, trace_id)
+ probability = _highest_yes_probability(team_id, {"question": question}, {"q": _ASKS_QUESTION}, trace_id)
return None if probability is None else probability >= ASKS_FOR_IDENTITY_PROBABILITY
@@ -127,7 +166,8 @@ def contains_pii(*, team_id: int, text: dict[str, str], trace_id: str) -> bool |
"""Whether the answer's text holds personal data; None when Jev could not answer."""
if not text:
return False
- probability = _yes_probability(team_id, {"text": text}, _DETECT_QUESTION, trace_id)
+ questions = {"name": _NAME_QUESTION, "contact": _CONTACT_QUESTION}
+ probability = _highest_yes_probability(team_id, {"text": text}, questions, trace_id)
return None if probability is None else probability >= PII_FLAG_PROBABILITY
diff --git a/products/replay_vision/backend/tests/test_pii_check.py b/products/replay_vision/backend/tests/test_pii_check.py
index 606c6a18eeda..521104c15f16 100644
--- a/products/replay_vision/backend/tests/test_pii_check.py
+++ b/products/replay_vision/backend/tests/test_pii_check.py
@@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, patch
from google.genai import errors as genai_errors
from parameterized import parameterized
+from products.ml_inference.backend.facade.contracts import DecisionGatewayError, DecisionResult, NoulAnswer
from products.replay_vision.backend.error_kinds import FailureKind
from products.replay_vision.backend.temporal import pii_check
from products.replay_vision.backend.temporal.errors import ScannerFailureError
@@ -81,3 +82,34 @@ async def test_a_provider_refusal_of_the_rewrite_fails_the_observation() -> None
with pytest.raises(ScannerFailureError) as raised:
await _check(f"(t 12) {_SUBJECT} paid.", False, rewrite_error=_refusal("googleapis.com"))
assert raised.value.kind == FailureKind.PII_DETECTED
+
+
+ [
+ ("name_only", 0.9, 0.1, True),
+ ("contact_only", 0.1, 0.9, True),
+ ("neither", 0.1, 0.1, False),
+ ("both_just_below_the_bar", 0.39, 0.39, False),
+ ]
+)
+def test_contains_pii_flags_when_either_question_clears_the_bar(
+ _name: str, name: float, contact: float, expected: bool
+) -> None:
+ result = DecisionResult(
+ model=pii_check.PII_CHECK_MODEL,
+ answers={"name": NoulAnswer(probability=name), "contact": NoulAnswer(probability=contact)},
+ input_tokens=1,
+ )
+ with (
+ patch.object(pii_check.decision_api, "decisions_available_here", return_value=True),
+ patch.object(pii_check.decision_api, "decide_when_available", return_value=result),
+ ):
+ assert pii_check.contains_pii(team_id=1, text={"reasoning": _CLEAN}, trace_id="t") is expected
+
+
+def test_contains_pii_is_unknown_when_the_gateway_fails() -> None:
+ with (
+ patch.object(pii_check.decision_api, "decisions_available_here", return_value=True),
+ patch.object(pii_check.decision_api, "decide_when_available", side_effect=DecisionGatewayError(503, "busy")),
+ ):
+ assert pii_check.contains_pii(team_id=1, text={"reasoning": _CLEAN}, trace_id="t") is None