Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ai): capture context selection outcomes as an event (#122694)

Adam Bowkerc88657b6cfb7
diff --git a/products/context_layer/backend/selection.py b/products/context_layer/backend/selection.py
index c6edc6263e31..4ad17c3d55c3 100644
--- a/products/context_layer/backend/selection.py
+++ b/products/context_layer/backend/selection.py
@@ -1,5 +1,6 @@
import json
import time
+import hashlib
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor, wait
from contextvars import copy_context
@@ -14,8 +15,10 @@ from django.db import connections
import httpx
import structlog
+import posthoganalytics
from posthog.dataclasses import frozen
+from posthog.event_usage import groups
from posthog.helpers.trigram_search import MAX_SEARCH_LENGTH, normalize_search_term
from posthog.llm.system_one import NoulAnswer, NoulQuestion
from posthog.llm.system_one_client import build_system_one_client
@@ -26,6 +29,8 @@ from posthog.ph_client import get_feature_flag_or_none
from products.tasks.backend.facade.api import is_current_task_run_actor
from products.tasks.backend.models import Task, TaskRun
+from ee.hogai.utils.feature_flags import is_privacy_mode_enabled
+
if TYPE_CHECKING:
from posthog.llm.system_one_client import SystemOneClient
@@ -322,18 +327,58 @@ class ContextSelector:
return scored
def select(self, deadline: float) -> PreparedContext:
- if self.gate(deadline) <= 0.3:
+ gate = self.gate(deadline)
+ if gate <= 0.3:
+ self.capture_outcome("gate_skipped", gate, [])
return PreparedContext(selection_id=self.selection_id, mode=self.mode, reason="gate_skipped")
remaining = time_remaining(deadline)
candidates = self.retrieve(deadline - remaining / 2)
- context = self.render(self.rerank(candidates, deadline))
+ scored = self.rerank(candidates, deadline)
+ context = self.render(scored)
+ reason = "selected" if context else "empty"
+ self.capture_outcome(reason, gate, scored)
return PreparedContext(
selection_id=self.selection_id,
mode=self.mode,
context=context if self.mode == "treatment" else "",
- reason="selected" if context else "empty",
+ reason=reason,
+ )
+
+ def capture_outcome(self, reason: str, gate: float, scored: list[tuple[Candidate, float]]) -> None:
+ # The gateway drops System One inputs and answers from its events, so this is the only record of decisions.
+ # The event goes to PostHog's own analytics, so it carries no customer-written names.
+ privacy_mode = is_privacy_mode_enabled(self.run.team)
+ candidates: list[dict[str, object]] = []
+ for candidate, score in sorted(scored, key=lambda pair: -pair[1]):
+ entry: dict[str, object] = {"kind": candidate.kind, "score": round(score, 4)}
+ if not privacy_mode:
+ entry["ref"] = self.candidate_ref(candidate)
+ candidates.append(entry)
+ posthoganalytics.capture(
+ distinct_id=str(self.actor.distinct_id),
+ event="context selection outcome",
+ properties={
+ **self.properties,
+ "selection_id": self.selection_id,
+ "mode": self.mode,
+ "config_version": CONFIG_VERSION,
+ "runtime_version": self.selection.runtime_version,
+ "reason": reason,
+ "gate_probability": round(gate, 4),
+ "candidate_count": len(scored),
+ "privacy_mode": privacy_mode,
+ "candidates": candidates,
+ },
+ groups=groups(self.run.team.organization, self.run.team),
)
+ def candidate_ref(self, candidate: Candidate) -> str:
+ if candidate.kind != "skill":
+ return candidate.id
+ # A skill's id is its customer-written name, so only a team-scoped digest leaves the process.
+ digest = hashlib.sha256(f"{self.run.team_id}:{candidate.id}".encode()).hexdigest()
+ return f"sha256:{digest[:16]}"
+
def render(self, scored: list[tuple[Candidate, float]]) -> str:
header = (
f'<posthog_reference_context selection_id="{self.selection_id}">\n'
diff --git a/products/context_layer/backend/test/test_selection.py b/products/context_layer/backend/test/test_selection.py
index b315d654e0ad..7727c5f301a2 100644
--- a/products/context_layer/backend/test/test_selection.py
+++ b/products/context_layer/backend/test/test_selection.py
@@ -337,6 +337,26 @@ class TestSelection(SimpleTestCase):
self.assertEqual(context.count("</posthog_reference_context>"), 1)
self.assertIn("\\u003c", context)
+ @parameterized.expand([(False,), (True,)])
+ def test_outcome_event_carries_no_customer_names(self, privacy_mode: bool) -> None:
+ self.selector.mode = "shadow"
+ metric_id = str(uuid4())
+ scored = [
+ (Candidate(id="Secret skill", kind="skill", title="Secret skill", text="t", reference="r"), 0.2),
+ (Candidate(id=metric_id, kind="metric", title="Secret metric", text="t", reference="r"), 0.9),
+ ]
+ with (
+ patch("products.context_layer.backend.selection.is_privacy_mode_enabled", return_value=privacy_mode),
+ patch("products.context_layer.backend.selection.posthoganalytics.capture") as capture,
+ ):
+ self.selector.capture_outcome("selected", 0.8, scored)
+ properties = capture.call_args.kwargs["properties"]
+ self.assertEqual(properties["reason"], "selected")
+ self.assertEqual([c["score"] for c in properties["candidates"]], [0.9, 0.2])
+ serialized = json.dumps(properties)
+ self.assertNotIn("Secret", serialized)
+ self.assertEqual(metric_id in serialized, not privacy_mode)
+
class TestSelectionPermissions(SimpleTestCase):
def test_session_credentials_cannot_prepare_context(self) -> None: