:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ai): capture context selection outcomes as an event (#122694)
Adam Bowkerc88657b6cfb7
diff --git a/products/context_layer/backend/selection.py b/products/context_layer/backend/selection.pyindex c6edc6263e31..4ad17c3d55c3 100644--- a/products/context_layer/backend/selection.py+++ b/products/context_layer/backend/selection.py@@ -1,5 +1,6 @@import jsonimport time+import hashlibfrom collections.abc import Callablefrom concurrent.futures import Future, ThreadPoolExecutor, waitfrom contextvars import copy_context@@ -14,8 +15,10 @@ from django.db import connectionsimport httpximport structlog+import posthoganalyticsfrom posthog.dataclasses import frozen+from posthog.event_usage import groupsfrom posthog.helpers.trigram_search import MAX_SEARCH_LENGTH, normalize_search_termfrom posthog.llm.system_one import NoulAnswer, NoulQuestionfrom posthog.llm.system_one_client import build_system_one_client@@ -26,6 +29,8 @@ from posthog.ph_client import get_feature_flag_or_nonefrom products.tasks.backend.facade.api import is_current_task_run_actorfrom products.tasks.backend.models import Task, TaskRun+from ee.hogai.utils.feature_flags import is_privacy_mode_enabled+if TYPE_CHECKING:from posthog.llm.system_one_client import SystemOneClient@@ -322,18 +327,58 @@ class ContextSelector:return scoreddef select(self, deadline: float) -> PreparedContext:- if self.gate(deadline) <= 0.3:+ gate = self.gate(deadline)+ if gate <= 0.3:+ self.capture_outcome("gate_skipped", gate, [])return PreparedContext(selection_id=self.selection_id, mode=self.mode, reason="gate_skipped")remaining = time_remaining(deadline)candidates = self.retrieve(deadline - remaining / 2)- context = self.render(self.rerank(candidates, deadline))+ scored = self.rerank(candidates, deadline)+ context = self.render(scored)+ reason = "selected" if context else "empty"+ self.capture_outcome(reason, gate, scored)return PreparedContext(selection_id=self.selection_id,mode=self.mode,context=context if self.mode == "treatment" else "",- reason="selected" if context else "empty",+ reason=reason,+ )++ def capture_outcome(self, reason: str, gate: float, scored: list[tuple[Candidate, float]]) -> None:+ # The gateway drops System One inputs and answers from its events, so this is the only record of decisions.+ # The event goes to PostHog's own analytics, so it carries no customer-written names.+ privacy_mode = is_privacy_mode_enabled(self.run.team)+ candidates: list[dict[str, object]] = []+ for candidate, score in sorted(scored, key=lambda pair: -pair[1]):+ entry: dict[str, object] = {"kind": candidate.kind, "score": round(score, 4)}+ if not privacy_mode:+ entry["ref"] = self.candidate_ref(candidate)+ candidates.append(entry)+ posthoganalytics.capture(+ distinct_id=str(self.actor.distinct_id),+ event="context selection outcome",+ properties={+ **self.properties,+ "selection_id": self.selection_id,+ "mode": self.mode,+ "config_version": CONFIG_VERSION,+ "runtime_version": self.selection.runtime_version,+ "reason": reason,+ "gate_probability": round(gate, 4),+ "candidate_count": len(scored),+ "privacy_mode": privacy_mode,+ "candidates": candidates,+ },+ groups=groups(self.run.team.organization, self.run.team),)+ def candidate_ref(self, candidate: Candidate) -> str:+ if candidate.kind != "skill":+ return candidate.id+ # A skill's id is its customer-written name, so only a team-scoped digest leaves the process.+ digest = hashlib.sha256(f"{self.run.team_id}:{candidate.id}".encode()).hexdigest()+ return f"sha256:{digest[:16]}"+def render(self, scored: list[tuple[Candidate, float]]) -> str:header = (f'<posthog_reference_context selection_id="{self.selection_id}">\n'diff --git a/products/context_layer/backend/test/test_selection.py b/products/context_layer/backend/test/test_selection.pyindex b315d654e0ad..7727c5f301a2 100644--- a/products/context_layer/backend/test/test_selection.py+++ b/products/context_layer/backend/test/test_selection.py@@ -337,6 +337,26 @@ class TestSelection(SimpleTestCase):self.assertEqual(context.count("</posthog_reference_context>"), 1)self.assertIn("\\u003c", context)+ @parameterized.expand([(False,), (True,)])+ def test_outcome_event_carries_no_customer_names(self, privacy_mode: bool) -> None:+ self.selector.mode = "shadow"+ metric_id = str(uuid4())+ scored = [+ (Candidate(id="Secret skill", kind="skill", title="Secret skill", text="t", reference="r"), 0.2),+ (Candidate(id=metric_id, kind="metric", title="Secret metric", text="t", reference="r"), 0.9),+ ]+ with (+ patch("products.context_layer.backend.selection.is_privacy_mode_enabled", return_value=privacy_mode),+ patch("products.context_layer.backend.selection.posthoganalytics.capture") as capture,+ ):+ self.selector.capture_outcome("selected", 0.8, scored)+ properties = capture.call_args.kwargs["properties"]+ self.assertEqual(properties["reason"], "selected")+ self.assertEqual([c["score"] for c in properties["candidates"]], [0.9, 0.2])+ serialized = json.dumps(properties)+ self.assertNotIn("Secret", serialized)+ self.assertEqual(metric_id in serialized, not privacy_mode)+class TestSelectionPermissions(SimpleTestCase):def test_session_credentials_cannot_prepare_context(self) -> None: