Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(taxonomic-filter): remove the event match endpoint and evals (#124142)

Paul D'Ambraa2266c05d77a
diff --git a/frontend/src/generated/core/api.schemas.ts b/frontend/src/generated/core/api.schemas.ts
index cc2cd50ecf3e..cbd454badf03 100644
--- a/frontend/src/generated/core/api.schemas.ts
+++ b/frontend/src/generated/core/api.schemas.ts
@@ -5542,61 +5542,6 @@ export interface SearchIntentResponseApi {
model_query: string | null
}
-export interface EventMatchRequestApi {
- /**
- * What the person typed into the events list search box, which matched no event name.
- * @maxLength 200
- */
- query: string
-}
-
-export interface EventMatchApi {
- /** The event name to select, such as $autocapture. */
- name: string
- /** The event's display name, such as Autocapture. */
- display_name: string
- /** How likely the search means this event, from 0 to 1. */
- probability: number
-}
-
-/**
- * * `wrong_length` - The search is too short or too long to ask about
- * * `only_values` - Nothing but values is left for the model to read
- * * `unavailable` - The model could not answer at all
- * * `partial` - Some model requests failed, so an event it never asked about might have matched
- * * `nothing_likely` - The model found no core event likely
- * * `not_ingested` - The project never sent the likely events
- * * `matched` - At least one likely event has data
- */
-export type EventMatchOutcomeEnumApi = (typeof EventMatchOutcomeEnumApi)[keyof typeof EventMatchOutcomeEnumApi]
-
-export const EventMatchOutcomeEnumApi = {
- WrongLength: 'wrong_length',
- OnlyValues: 'only_values',
- Unavailable: 'unavailable',
- Partial: 'partial',
- NothingLikely: 'nothing_likely',
- NotIngested: 'not_ingested',
- Matched: 'matched',
-} as const
-
-export interface EventMatchResponseApi {
- /** PostHog core events the search most likely means, strongest first. Empty when nothing is likely. */
- matches: EventMatchApi[]
- /** The step that decided the answer, so an empty answer says why it is empty.
- *
- * * `wrong_length` - The search is too short or too long to ask about
- * * `only_values` - Nothing but values is left for the model to read
- * * `unavailable` - The model could not answer at all
- * * `partial` - Some model requests failed, so an event it never asked about might have matched
- * * `nothing_likely` - The model found no core event likely
- * * `not_ingested` - The project never sent the likely events
- * * `matched` - At least one likely event has data */
- outcome: EventMatchOutcomeEnumApi
- /** Likely core events this project never sent, strongest first. Only filled when outcome is not_ingested. */
- unsent: EventMatchApi[]
-}
-
/**
* * `offer` - offer
* * `answer` - answer
diff --git a/frontend/src/generated/core/api.ts b/frontend/src/generated/core/api.ts
index 8ce22cc15922..f5a52cb18024 100644
--- a/frontend/src/generated/core/api.ts
+++ b/frontend/src/generated/core/api.ts
@@ -27,8 +27,6 @@ import type {
EmojiSearchSuggestRetrieveParams,
EnterprisePropertyDefinitionApi,
EventIngestionRestrictionApi,
- EventMatchRequestApi,
- EventMatchResponseApi,
EventPropertyValuesResponseApi,
EventsValuesRetrieveParams,
ExportedAssetApi,
@@ -2764,27 +2762,6 @@ export const taxonomicSearchIntentClassifyCreate = async (
})
}
-export const getTaxonomicSearchIntentMatchEventsCreateUrl = (projectId: string) => {
- return `/api/projects/${projectId}/taxonomic_search_intent/match_events/`
-}
-
-/**
- * Guess which PostHog core events a search that matched no event name describes.
- * @summary Match an events search to core events
- */
-export const taxonomicSearchIntentMatchEventsCreate = async (
- projectId: string,
- eventMatchRequestApi: EventMatchRequestApi,
- options?: RequestInit
-): Promise<EventMatchResponseApi> => {
- return apiMutator<EventMatchResponseApi>(getTaxonomicSearchIntentMatchEventsCreateUrl(projectId), {
- ...options,
- method: 'POST',
- headers: { 'Content-Type': 'application/json', ...options?.headers },
- body: JSON.stringify(eventMatchRequestApi),
- })
-}
-
export const getTerminalNetplayMailboxRetrieveUrl = (
projectId: string,
params: TerminalNetplayMailboxRetrieveParams
diff --git a/frontend/src/generated/core/api.zod.ts b/frontend/src/generated/core/api.zod.ts
index 8bcba1098eff..c7a106bb05e3 100644
--- a/frontend/src/generated/core/api.zod.ts
+++ b/frontend/src/generated/core/api.zod.ts
@@ -1217,19 +1217,6 @@ export const TaxonomicSearchIntentClassifyCreateBody = /* @__PURE__ */ zod.objec
.describe('The id of the scene the picker is open in, such as Insight or Replay.'),
})
-/**
- * Guess which PostHog core events a search that matched no event name describes.
- * @summary Match an events search to core events
- */
-export const taxonomicSearchIntentMatchEventsCreateBodyQueryMax = 200
-
-export const TaxonomicSearchIntentMatchEventsCreateBody = /* @__PURE__ */ zod.object({
- query: zod
- .string()
- .max(taxonomicSearchIntentMatchEventsCreateBodyQueryMax)
- .describe('What the person typed into the events list search box, which matched no event name.'),
-})
-
/**
* Send a WebRTC session description to another terminal in a Doom room.
*/
diff --git a/posthog/api/taxonomic_search_intent.py b/posthog/api/taxonomic_search_intent.py
index 94851217cdf8..341e67f79d1b 100644
--- a/posthog/api/taxonomic_search_intent.py
+++ b/posthog/api/taxonomic_search_intent.py
@@ -17,18 +17,10 @@ from rest_framework.throttling import UserRateThrottle
from posthog.api.mixins import validated_request
from posthog.api.routing import TeamAndOrgViewSetMixin
-from posthog.event_usage import report_user_action
from posthog.llm.system_one import SystemOneNotConfigured, SystemOneRequestFailed
from posthog.models import User
from posthog.taxonomic_search_intent.classify import classify_search_intent, search_intent_enabled
-from posthog.taxonomic_search_intent.contracts import (
- EventMatch,
- EventMatchOutcome,
- EventMatchRequest,
- SearchIntentRequest,
- SearchIntentSource,
-)
-from posthog.taxonomic_search_intent.event_match import EVENT_MATCH_FEATURE_FLAG, match_core_events
+from posthog.taxonomic_search_intent.contracts import SearchIntentRequest, SearchIntentSource
logger = structlog.get_logger(__name__)
@@ -95,40 +87,6 @@ class SearchIntentResponseSerializer(serializers.Serializer):
)
-class EventMatchRequestSerializer(serializers.Serializer):
- query = serializers.CharField(
- max_length=MAX_SEARCH_QUERY_CHARS,
- allow_blank=True,
- trim_whitespace=True,
- help_text="What the person typed into the events list search box, which matched no event name.",
- )
-
-
-class EventMatchSerializer(serializers.Serializer):
- name = serializers.CharField(help_text="The event name to select, such as $autocapture.")
- display_name = serializers.CharField(help_text="The event's display name, such as Autocapture.")
- probability = serializers.FloatField(help_text="How likely the search means this event, from 0 to 1.")
-
-
-class EventMatchResponseSerializer(serializers.Serializer):
- matches = serializers.ListField(
- child=EventMatchSerializer(),
- help_text="PostHog core events the search most likely means, strongest first. Empty when nothing is likely.",
- )
- outcome = serializers.ChoiceField(
- choices=EventMatchOutcome.choices,
- help_text="The step that decided the answer, so an empty answer says why it is empty.",
- )
- unsent = serializers.ListField(
- child=EventMatchSerializer(),
- help_text="Likely core events this project never sent, strongest first. Only filled when outcome is not_ingested.",
- )
-
-
-def _serialize_match(match: EventMatch) -> dict[str, Any]:
- return {"name": match.name, "display_name": match.label, "probability": match.probability}
-
-
class SearchIntentUnavailable(APIException):
status_code = status.HTTP_503_SERVICE_UNAVAILABLE
default_detail = "The decision model is not available right now. Try again in a moment."
@@ -147,18 +105,6 @@ class SearchIntentSustainedThrottle(UserRateThrottle):
rate = "3000/day"
-# Each event match miss makes several model requests, and it only runs for a search with no results,
-# so it gets far lower rates than classify.
-class EventMatchBurstThrottle(UserRateThrottle):
- scope = "taxonomic_event_match_burst"
- rate = "20/minute"
-
-
-class EventMatchSustainedThrottle(UserRateThrottle):
- scope = "taxonomic_event_match_sustained"
- rate = "400/day"
-
-
@extend_schema(extensions={"x-product": "core"})
class SearchIntentViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
scope_object = "INTERNAL"
@@ -207,48 +153,3 @@ class SearchIntentViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
}
).data
)
-
- @validated_request(
- request_serializer=EventMatchRequestSerializer,
- responses={
- 200: OpenApiResponse(response=EventMatchResponseSerializer, description="The likely core events."),
- 404: OpenApiResponse(description="Event matching is not enabled for this person."),
- 503: OpenApiResponse(description="The decision model did not answer in time."),
- },
- summary="Match an events search to core events",
- description="Guess which PostHog core events a search that matched no event name describes.",
- )
- @action(detail=False, methods=["POST"], throttle_classes=[EventMatchBurstThrottle, EventMatchSustainedThrottle])
- def match_events(self, request: Request, **kwargs: Any) -> Response:
- user = cast(User, request.user)
- if not search_intent_enabled(str(user.distinct_id), str(self.organization_id), flag=EVENT_MATCH_FEATURE_FLAG):
- raise NotFound()
- search = EventMatchRequest(
- team_id=self.team_id, project_id=self.team.project_id, query=request.validated_data["query"]
- )
- try:
- answer = match_core_events(search)
- except (SystemOneNotConfigured, SystemOneRequestFailed) as error:
- logger.warning("taxonomic_event_match_unavailable", team_id=self.team_id, reason=type(error).__name__)
- self._report_event_match(request, EventMatchOutcome.UNAVAILABLE)
- raise SearchIntentUnavailable() from error
- self._report_event_match(request, answer.outcome)
- return Response(
- EventMatchResponseSerializer(
- {
- "matches": [_serialize_match(match) for match in answer.matches],
- "outcome": answer.outcome.value,
- "unsent": [_serialize_match(match) for match in answer.unsent],
- }
- ).data
- )
-
- def _report_event_match(self, request: Request, outcome: EventMatchOutcome) -> None:
- # The picker counts only the answers it shows, so only this event says which step left an answer empty.
- report_user_action(
- cast(User, request.user),
- "taxonomic filter event match answered",
- {"outcome": outcome.value},
- team=self.team,
- request=request,
- )
diff --git a/posthog/api/test/test_taxonomic_search_intent.py b/posthog/api/test/test_taxonomic_search_intent.py
index 154f22019749..fe57c270cfa0 100644
--- a/posthog/api/test/test_taxonomic_search_intent.py
+++ b/posthog/api/test/test_taxonomic_search_intent.py
@@ -9,7 +9,6 @@ from rest_framework import status
from posthog.llm.gateway_client import team_distinct_id
from posthog.llm.system_one import ChoiceAnswer, SystemOneNotConfigured, SystemOneRequestFailed, SystemOneResult
-from posthog.taxonomic_search_intent.contracts import EventMatch, EventMatchAnswer, EventMatchOutcome
from posthog.taxonomic_search_intent.prompt import BUNDLED_SEARCH_INTENT_PROMPT
ALL_TABS = ("suggested_filters", "events", "event_properties", "person_properties", "pageview_urls", "email_addresses")
@@ -100,83 +99,3 @@ class TestSearchIntentEndpoint(APIBaseTest):
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
-
- @parameterized.expand(
- [
- ("enabled", True, None, status.HTTP_200_OK, "matched"),
- ("flag_off", False, None, status.HTTP_404_NOT_FOUND, None),
- (
- "model_unreachable",
- True,
- SystemOneRequestFailed("timeout"),
- status.HTTP_503_SERVICE_UNAVAILABLE,
- "unavailable",
- ),
- ]
- )
- def test_matches_events_behind_its_own_flag(self, _name, enabled, error, expected_status, expected_outcome) -> None:
- match = EventMatch(name="$autocapture", label="Autocapture", probability=0.95)
- answer = EventMatchAnswer(matches=[match], outcome=EventMatchOutcome.MATCHED)
- with (
- patch(FLAG_CHECK, return_value=enabled) as flag,
- patch(
- "posthog.api.taxonomic_search_intent.match_core_events", return_value=answer, side_effect=error
- ) as matcher,
- patch("posthog.api.taxonomic_search_intent.report_user_action") as report,
- ):
- response = self.client.post(
- f"/api/projects/{self.team.id}/taxonomic_search_intent/match_events/",
- {"query": "browser capture"},
- format="json",
- )
-
- assert response.status_code == expected_status
- assert flag.call_args.args[0] == "taxonomic-filter-event-match"
- if expected_status == status.HTTP_200_OK:
- assert response.json() == {
- "matches": [{"name": "$autocapture", "display_name": "Autocapture", "probability": 0.95}],
- "outcome": "matched",
- "unsent": [],
- }
- assert matcher.call_args.args[0].project_id == self.team.project_id
- if expected_outcome:
- assert report.call_args.args[1:] == ("taxonomic filter event match answered", {"outcome": expected_outcome})
- else:
- report.assert_not_called()
-
- @parameterized.expand(
- [
- (
- "not_ingested",
- EventMatchAnswer(
- matches=[],
- outcome=EventMatchOutcome.NOT_INGESTED,
- unsent=[EventMatch(name="$screen", label="Screen", probability=0.85)],
- ),
- {
- "matches": [],
- "outcome": "not_ingested",
- "unsent": [{"name": "$screen", "display_name": "Screen", "probability": 0.85}],
- },
- ),
- (
- "only_values",
- EventMatchAnswer(matches=[], outcome=EventMatchOutcome.ONLY_VALUES),
- {"matches": [], "outcome": "only_values", "unsent": []},
- ),
- ]
- )
- def test_tells_the_picker_why_an_answer_is_empty(self, _name, answer, expected) -> None:
- with (
- patch(FLAG_CHECK, return_value=True),
- patch("posthog.api.taxonomic_search_intent.match_core_events", return_value=answer),
- patch("posthog.api.taxonomic_search_intent.report_user_action"),
- ):
- response = self.client.post(
- f"/api/projects/{self.team.id}/taxonomic_search_intent/match_events/",
- {"query": "browser capture"},
- format="json",
- )
-
- assert response.status_code == status.HTTP_200_OK
- assert response.json() == expected
diff --git a/posthog/taxonomic_search_intent/classify.py b/posthog/taxonomic_search_intent/classify.py
index 1bcc564d70e4..7295471d77d7 100644
--- a/posthog/taxonomic_search_intent/classify.py
+++ b/posthog/taxonomic_search_intent/classify.py
@@ -62,7 +62,7 @@ _QUESTION_ID = "tab"
_CACHED_INTENT = TypeAdapter(SearchIntent)
-def search_intent_enabled(distinct_id: str, organization_id: str, flag: str = SEARCH_INTENT_FEATURE_FLAG) -> bool:
+def search_intent_enabled(distinct_id: str, organization_id: str) -> bool:
"""Any flag value other than off enables the endpoint, because every arm of an experiment asks the model.
Dark launch: only local development and the US cloud, so no flag change can bring it up in the EU or on
@@ -75,7 +75,7 @@ def search_intent_enabled(distinct_id: str, organization_id: str, flag: str = SE
try:
return bool(
posthoganalytics.feature_enabled(
- flag,
+ SEARCH_INTENT_FEATURE_FLAG,
distinct_id,
groups={"organization": organization_id},
group_properties={"organization": {"id": organization_id}},
diff --git a/posthog/taxonomic_search_intent/contracts.py b/posthog/taxonomic_search_intent/contracts.py
index 3fa9ba3ae13e..e53ec5bccf37 100644
--- a/posthog/taxonomic_search_intent/contracts.py
+++ b/posthog/taxonomic_search_intent/contracts.py
@@ -1,5 +1,3 @@
-from dataclasses import field
-
from django.db import models
from posthog.dataclasses import frozen
@@ -11,16 +9,6 @@ class SearchIntentSource(models.TextChoices):
SKIPPED = "skipped", "Not classified"
-class EventMatchOutcome(models.TextChoices):
- WRONG_LENGTH = "wrong_length", "The search is too short or too long to ask about"
- ONLY_VALUES = "only_values", "Nothing but values is left for the model to read"
- UNAVAILABLE = "unavailable", "The model could not answer at all"
- PARTIAL = "partial", "Some model requests failed, so an event it never asked about might have matched"
- NOTHING_LIKELY = "nothing_likely", "The model found no core event likely"
- NOT_INGESTED = "not_ingested", "The project never sent the likely events"
- MATCHED = "matched", "At least one likely event has data"
-
-
@frozen
class SearchIntentRequest:
"""What a person typed into the filter picker, and the tabs the picker shows them."""
@@ -45,32 +33,3 @@ class SearchIntent:
prompt_version: int | None = None
# The search as the model read it, with values replaced by placeholders. None when the model did not answer.
model_query: str | None = None
-
-
-@frozen
-class EventMatchRequest:
- """A search in the picker's events list that found nothing, to match against the PostHog core events."""
-
- team_id: int
- project_id: int
- query: str
-
-
-@frozen
-class EventMatch:
- """A core event the search may mean. ``probability`` is the model's confidence that the search names it."""
-
- name: str
- label: str
- probability: float
-
-
-@frozen
-class EventMatchAnswer:
- """The suggestions for a search, and the step that decided them, so an empty answer says why it is empty."""
-
- matches: list[EventMatch]
- outcome: EventMatchOutcome
- # Likely core events the project never sent, strongest first. Filled only for NOT_INGESTED, so the picker can say
- # what the search probably means instead of showing nothing.
- unsent: list[EventMatch] = field(default_factory=list)
diff --git a/posthog/taxonomic_search_intent/event_match.py b/posthog/taxonomic_search_intent/event_match.py
deleted file mode 100644
index ac1ea91114f7..000000000000
--- a/posthog/taxonomic_search_intent/event_match.py
+++ /dev/null
@@ -1,240 +0,0 @@
-"""
-Match a filter picker search that found no events to the PostHog core events it may mean.
-
-A person who does not know an event's name types what the event does, such as "browser capture" for
-autocapture. Fuzzy search cannot bridge that gap, so each core event becomes one yes/no question to the
-decision model, with the event's label and description as the meaning to judge.
-"""
-
-import hashlib
-import contextvars
-from collections.abc import Sequence
-from concurrent.futures import ThreadPoolExecutor
-
-from django.core.cache import cache
-
-import structlog
-from pydantic import TypeAdapter, ValidationError
-
-from posthog.dataclasses import frozen
-from posthog.exceptions_capture import capture_exception
-from posthog.llm.gateway_client import team_distinct_id
-from posthog.llm.managed_decision_model import ManagedDecisionModel
-from posthog.llm.system_one import NoulAnswer, NoulQuestion, SystemOneRequestFailed, SystemOneResult
-from posthog.llm.system_one_client import GATEWAY_MAX_QUESTIONS, SystemOneClient, build_system_one_client
-from posthog.models import EventDefinition
-from posthog.taxonomy.taxonomy import CORE_FILTER_DEFINITIONS_BY_GROUP
-
-from .classify import CACHE_TTL_SECONDS, MAX_QUERY_CHARS, MIN_QUERY_CHARS, SEARCH_INTENT_TIMEOUT_SECONDS, redact_values
-from .contracts import EventMatch, EventMatchAnswer, EventMatchOutcome, EventMatchRequest
-
-logger = structlog.get_logger(__name__)
-
-EVENT_MATCH_FEATURE_FLAG = "taxonomic-filter-event-match"
-# The picker shows a suggestion only above this probability. Tune it from the eval suite, not by feel.
-MATCH_THRESHOLD = 0.7
-# The picker shows at most this many, after it removes the events it excludes, so the endpoint returns them all.
-MAX_MATCHES = 3
-CACHE_KEY_PREFIX = "taxonomic_search_intent:event_match:v1"
-EVENT_MATCH_MODEL = ManagedDecisionModel("taxonomic-filter-event-match")
-
-_CACHED_MATCHES = TypeAdapter(list[EventMatch])
-
-
-@frozen
-class CoreEventCandidate:
- label: str
- # The description the model judges the search against.
- meaning: str
-
-
-def _candidates() -> dict[str, CoreEventCandidate]:
- """Every core event a picker can select, by event name."""
- candidates: dict[str, CoreEventCandidate] = {}
- for name, definition in CORE_FILTER_DEFINITIONS_BY_GROUP["events"].items():
- # "All events" is not an event. Events hidden from query builders stay in: the picker drops them from
- # the suggestions when it hides them, which depends on flags only the frontend knows.
- if name == "All events":
- continue
- meaning = definition.get("description_llm") or definition.get("description") or ""
- candidates[name] = CoreEventCandidate(label=definition["label"], meaning=meaning)
- return candidates
-
-
-CORE_EVENT_CANDIDATES = _candidates()
-_CANDIDATES_DIGEST = hashlib.sha256(repr(sorted(CORE_EVENT_CANDIDATES.items())).encode()).hexdigest()[:16]
-
-
-def event_match_state(query: str) -> str:
- return "\n".join(
- [
- "A person searches the events list of the filter picker in PostHog, a product analytics tool.",
- "No event name matches the search, so the person may describe an event instead of naming it.",
- f"Search: {query}",
- ]
- )
-
-
-def _question(candidate: CoreEventCandidate) -> NoulQuestion:
- return NoulQuestion(
- instructions=f"Does the search look for the {candidate.label} event? {candidate.meaning}".strip(),
- criteria_true="The search describes this event, uses a synonym for it, or names what it records.",
- criteria_false="The search is about something else.",
- )
-
-
-def _ask(client: SystemOneClient, state: str, names: Sequence[str]) -> SystemOneResult:
- questions = {f"e{index}": _question(CORE_EVENT_CANDIDATES[name]) for index, name in enumerate(names)}
- return client.decide(state=state, questions=questions)
-
-
-@frozen
-class _ModelAnswers:
- probabilities: dict[str, float]
- # False when some requests failed, so the probabilities cover only part of the core events.
- complete: bool
-
-
-def _ask_chunk(client: SystemOneClient, state: str, names: Sequence[str], team_id: int) -> SystemOneResult | None:
- """None when this request fails. The failure goes to error tracking and the logs, and the other chunks still count."""
- try:
- return _ask(client, state, names)
- except SystemOneRequestFailed as error:
- # Only the team and the failure go out: the error message never carries the search text.
- logger.warning(
- "taxonomic_event_match_chunk_failed",
- team_id=team_id,
- status_code=error.status_code,
- chunk_size=len(names),
- )
- capture_exception(error, {"team_id": team_id, "chunk_size": len(names)})
- return None
-
-
-def _probabilities(team_id: int, query: str, model: str) -> _ModelAnswers:
- """The model's probability for every core event it answered. Raises the System One errors when no request succeeds."""
- names = list(CORE_EVENT_CANDIDATES)
- # The gateway takes a bounded number of questions per request, so the candidates go out in parallel chunks.
- chunks = [names[start : start + GATEWAY_MAX_QUESTIONS] for start in range(0, len(names), GATEWAY_MAX_QUESTIONS)]
- client = build_system_one_client(
- model=model,
- ai_product="taxonomic_filter",
- distinct_id=team_distinct_id(team_id),
- timeout=SEARCH_INTENT_TIMEOUT_SECONDS,
- )
- state = event_match_state(query)
- with ThreadPoolExecutor(max_workers=len(chunks), thread_name_prefix="event-match") as executor:
- # A fresh copy of the request context per chunk keeps its log and error-tracking tags on the failure reports.
- futures = [
- executor.submit(contextvars.copy_context().run, _ask_chunk, client, state, chunk, team_id)
- for chunk in chunks
- ]
- results = [future.result() for future in futures]
- if all(result is None for result in results):
- raise SystemOneRequestFailed("Every event match request failed")
- probabilities: dict[str, float] = {}
- for chunk, result in zip(chunks, results):
- if result is None:
- continue
- for index, name in enumerate(chunk):
- answer = result.answers[f"e{index}"]
- if isinstance(answer, NoulAnswer):
- probabilities[name] = answer.probability
- return _ModelAnswers(probabilities=probabilities, complete=None not in results)
-
-
-def _cache_key(team_id: int, query: str, model: str) -> str:
- # Keyed per team, so a fast answer cannot tell one team what another team searched.
- digest = hashlib.sha256(f"{model}\n{MATCH_THRESHOLD}\n{query}".encode()).hexdigest()
- return f"{CACHE_KEY_PREFIX}:{_CANDIDATES_DIGEST}:{team_id}:{digest}"
-
-
-def likely_core_events(
- team_id: int, query: str, *, use_cache: bool = True, require_complete: bool = False, model: str | None = None
-) -> list[EventMatch]:
- """Every core event the model finds likely, strongest first, before the check against ingested events.
-
- The model reads the search with its values replaced by placeholders, and nothing when only values are left.
- With `require_complete`, a failed request raises instead of leaving its events out of the answer.
- """
- model_query = redact_values(query)
- if model_query is None:
- return []
- likely, _complete = _likely_core_events(
- team_id, model_query, use_cache=use_cache, require_complete=require_complete, model=model
- )
- return likely
-
-
-def _likely_core_events(
- team_id: int, model_query: str, *, use_cache: bool, require_complete: bool, model: str | None
-) -> tuple[list[EventMatch], bool]:
- """Same as `likely_core_events`, plus whether every core event was actually asked about.
-
- Takes the redacted query, so `match_core_events` can name the `ONLY_VALUES` outcome without redacting twice.
- A cache hit is always complete, since a partial answer is never cached (see below).
- """
- model = model or EVENT_MATCH_MODEL.current()
- key = _cache_key(team_id, model_query, model)
- if use_cache:
- cached = cache.get(key)
- # The Django cache pickles what it stores, so matches go in as JSON text and come out schema-validated.
- if isinstance(cached, str):
- try:
- return _CACHED_MATCHES.validate_json(cached), True
- except ValidationError:
- pass
- answers = _probabilities(team_id, model_query, model)
- if require_complete and not answers.complete:
- raise SystemOneRequestFailed("Some event match requests failed")
- likely = sorted(
- (
- EventMatch(name=name, label=CORE_EVENT_CANDIDATES[name].label, probability=probability)
- for name, probability in answers.probabilities.items()
- if probability >= MATCH_THRESHOLD
- ),
- key=lambda match: match.probability,
- reverse=True,
- )
- # A partial answer is not cached, so the next search asks again for the events that failed.
- if use_cache and answers.complete:
- cache.set(key, _CACHED_MATCHES.dump_json(likely).decode(), CACHE_TTL_SECONDS)
- return likely, answers.complete
-
-
-def _ingested(project_id: int, names: Sequence[str]) -> set[str]:
- return set(
- # A definition can exist before its first event arrives, and only a seen event has data to show.
- EventDefinition.objects.filter(
- team__project_id=project_id, name__in=names, last_seen_at__isnull=False
- ).values_list("name", flat=True)
- )
-
-
-def match_core_events(request: EventMatchRequest, *, use_cache: bool = True) -> EventMatchAnswer:
- """The core events the search most likely means, strongest first, limited to events the project has ingested.
-
- Raises the System One errors; the caller decides whether a failed answer matters.
- """
- query = " ".join(request.query.split())
- if not MIN_QUERY_CHARS <= len(query) <= MAX_QUERY_CHARS:
- return EventMatchAnswer(matches=[], outcome=EventMatchOutcome.WRONG_LENGTH)
- model_query = redact_values(query)
- if model_query is None:
- return EventMatchAnswer(matches=[], outcome=EventMatchOutcome.ONLY_VALUES)
- likely, complete = _likely_core_events(
- request.team_id, model_query, use_cache=use_cache, require_complete=False, model=None
- )
- if not likely:
- # A failed chunk leaves its events unasked, so an empty answer is not conclusive unless every chunk answered.
- return EventMatchAnswer(
- matches=[], outcome=EventMatchOutcome.NOTHING_LIKELY if complete else EventMatchOutcome.PARTIAL
- )
- # A suggestion for an event the project never sent would lead to an empty insight.
- ingested = _ingested(request.project_id, [match.name for match in likely])
- matches = [match for match in likely if match.name in ingested]
- if matches:
- return EventMatchAnswer(matches=matches, outcome=EventMatchOutcome.MATCHED)
- if not complete:
- return EventMatchAnswer(matches=[], outcome=EventMatchOutcome.PARTIAL)
- return EventMatchAnswer(matches=[], outcome=EventMatchOutcome.NOT_INGESTED, unsent=likely)
diff --git a/posthog/taxonomic_search_intent/test_event_match.py b/posthog/taxonomic_search_intent/test_event_match.py
deleted file mode 100644
index a265f108b463..000000000000
--- a/posthog/taxonomic_search_intent/test_event_match.py
+++ /dev/null
@@ -1,207 +0,0 @@
-from collections.abc import Mapping
-from datetime import UTC, datetime
-
-from posthog.test.base import BaseTest
-from unittest.mock import MagicMock, patch
-
-from django.core.cache import cache
-
-from parameterized import parameterized
-
-from posthog.llm.system_one import NoulAnswer, Question, SystemOneRequestFailed, SystemOneResult
-from posthog.llm.system_one_client import GATEWAY_MAX_QUESTIONS
-from posthog.models import EventDefinition
-from posthog.taxonomic_search_intent.contracts import EventMatch, EventMatchAnswer, EventMatchOutcome, EventMatchRequest
-from posthog.taxonomic_search_intent.event_match import (
- CORE_EVENT_CANDIDATES,
- _question,
- likely_core_events,
- match_core_events,
-)
-
-BUILD_CLIENT = "posthog.taxonomic_search_intent.event_match.build_system_one_client"
-CAPTURE = "posthog.taxonomic_search_intent.event_match.capture_exception"
-# Any past time works: the check only asks whether an event was ever seen.
-SEEN_AT = datetime(2026, 1, 1, tzinfo=UTC)
-
-
-# The last candidate is asked in a different request from Autocapture, so it survives Autocapture's request failing.
-LATE_EVENT_LABEL = CORE_EVENT_CANDIDATES[list(CORE_EVENT_CANDIDATES)[-1]].label
-
-LABEL_BY_INSTRUCTIONS = {
- _question(candidate).instructions: candidate.label for candidate in CORE_EVENT_CANDIDATES.values()
-}
-
-
-def _model_that_believes(probabilities: Mapping[str, float], failing_label: str | None = None) -> MagicMock:
- client = MagicMock()
-
- def decide(*, state: str, questions: Mapping[str, Question]) -> SystemOneResult:
- labels = {LABEL_BY_INSTRUCTIONS[question.instructions] for question in questions.values()}
- if failing_label in labels:
- raise SystemOneRequestFailed("The ai-gateway was not reached: ReadTimeout")
- answers = {
- question_id: NoulAnswer(probability=probabilities.get(LABEL_BY_INSTRUCTIONS[question.instructions], 0.01))
- for question_id, question in questions.items()
- }
- return SystemOneResult(model="jevk5-0.2", answers=answers, input_tokens=90)
-
- client.decide.side_effect = decide
- return client
-
-
-class TestMatchCoreEvents(BaseTest):
- def setUp(self) -> None:
- super().setUp()
- cache.clear()
- for name in ("$autocapture", "$rageclick", "$pageview", "$exception"):
- EventDefinition.objects.create(
- team=self.team, project_id=self.team.project_id, name=name, last_seen_at=SEEN_AT
- )
- # Defined but never seen, so it has no data to show.
- EventDefinition.objects.create(team=self.team, project_id=self.team.project_id, name="$screen")
-
- def _search(self, query: str, team_id: int | None = None) -> EventMatchRequest:
- return EventMatchRequest(team_id=team_id or self.team.id, project_id=self.team.project_id, query=query)
-
- @parameterized.expand(
- [
- ("too_short", "a", EventMatchOutcome.WRONG_LENGTH),
- ("too_long", "browser capture " * 5, EventMatchOutcome.WRONG_LENGTH),
- ("email", "[email protected]", EventMatchOutcome.ONLY_VALUES),
- ("url", "https://example.com/pricing", EventMatchOutcome.ONLY_VALUES),
- ("path", "/reset?token=abc", EventMatchOutcome.ONLY_VALUES),
- ]
- )
- def test_never_asks_the_model_about_values_or_unanswerable_searches(
- self, _name: str, query: str, outcome: EventMatchOutcome
- ) -> None:
- with patch(BUILD_CLIENT) as build:
- assert match_core_events(self._search(query)) == EventMatchAnswer(matches=[], outcome=outcome)
-
- build.assert_not_called()
-
- @parameterized.expand(
- [
- ("nothing_above_threshold", {"Autocapture": 0.6}, None, EventMatchOutcome.NOTHING_LIKELY),
- ("likely_but_never_seen", {"Screen": 0.85}, None, EventMatchOutcome.NOT_INGESTED),
- ("likely_and_seen", {"Screen": 0.85, "Autocapture": 0.95}, None, EventMatchOutcome.MATCHED),
- ("failed_chunk_and_nothing_likely", {}, "Autocapture", EventMatchOutcome.PARTIAL),
- ("failed_chunk_and_never_seen", {LATE_EVENT_LABEL: 0.9}, "Autocapture", EventMatchOutcome.PARTIAL),
- ]
- )
- def test_names_the_step_that_decided_the_answer(
- self, _name: str, beliefs: dict[str, float], failing_label: str | None, outcome: EventMatchOutcome
- ) -> None:
- with patch(BUILD_CLIENT, return_value=_model_that_believes(beliefs, failing_label)), patch(CAPTURE):
- answer = match_core_events(self._search("browser capture"))
-
- assert answer.outcome == outcome
- if outcome != EventMatchOutcome.MATCHED:
- assert answer.matches == []
- if outcome != EventMatchOutcome.NOT_INGESTED:
- assert answer.unsent == []
-
- def test_names_the_likely_events_the_project_never_sent(self) -> None:
- with patch(BUILD_CLIENT, return_value=_model_that_believes({"Screen": 0.85})):
- answer = match_core_events(self._search("browser capture"))
-
- assert answer == EventMatchAnswer(
- matches=[],
- outcome=EventMatchOutcome.NOT_INGESTED,
- unsent=[EventMatch(name="$screen", label="Screen", probability=0.85)],
- )
-
- def test_the_model_reads_placeholders_instead_of_values(self) -> None:
- client = _model_that_believes({})
- with patch(BUILD_CLIENT, return_value=client):
- match_core_events(self._search("visits https://example.com/reset?token=abc"))
-
- assert {call.kwargs["state"].splitlines()[-1] for call in client.decide.call_args_list} == {
- "Search: visits <url>"
- }
-
- def test_asks_about_every_core_event_within_the_gateway_limit(self) -> None:
- client = _model_that_believes({})
- with patch(BUILD_CLIENT, return_value=client):
- match_core_events(self._search("browser capture"))
-
- asked = [len(call.kwargs["questions"]) for call in client.decide.call_args_list]
- assert max(asked) <= GATEWAY_MAX_QUESTIONS
- assert sum(asked) == len(CORE_EVENT_CANDIDATES)
- assert "Search: browser capture" in client.decide.call_args.kwargs["state"]
-
- def test_model_change_recomputes_cached_matches(self) -> None:
- client = _model_that_believes({"Autocapture": 0.95})
- models = ["posthog/hogference/jevk5-fp8-0.2", "posthog/hogference/jeeves-0.1"]
- with (
- patch(BUILD_CLIENT, return_value=client) as build,
- patch("posthog.taxonomic_search_intent.event_match.EVENT_MATCH_MODEL.current", side_effect=models),
- ):
- match_core_events(self._search("browser capture"))
- match_core_events(self._search("browser capture"))
-
- assert build.call_count == 2
- assert [call.kwargs["model"] for call in build.call_args_list] == models
-
- def test_suggests_the_likely_ingested_events_strongest_first(self) -> None:
- client = _model_that_believes(
- {
- "Rageclick": 0.75,
- "Autocapture": 0.95,
- # Likely, but the project never sent it, so a suggestion would lead to an empty insight.
- "Dead click": 0.9,
- "Pageview": 0.72,
- "Exception": 0.71,
- "Screen": 0.85,
- }
- )
- with patch(BUILD_CLIENT, return_value=client):
- matches = match_core_events(self._search("browser capture")).matches
-
- assert matches == [
- EventMatch(name="$autocapture", label="Autocapture", probability=0.95),
- EventMatch(name="$rageclick", label="Rageclick", probability=0.75),
- EventMatch(name="$pageview", label="Pageview", probability=0.72),
- EventMatch(name="$exception", label="Exception", probability=0.71),
- ]
-
- @parameterized.expand([("same_team", False, 1), ("other_team", True, 2)])
- def test_the_same_search_is_answered_once_per_team(
- self, _name: str, other_team: bool, expected_requests: int
- ) -> None:
- second_team_id = self.team.id + 1 if other_team else None
- with patch(BUILD_CLIENT, return_value=_model_that_believes({"Autocapture": 0.95})) as build:
- match_core_events(self._search("browser capture"))
- match_core_events(self._search(" browser capture ", team_id=second_team_id))
-
- assert build.call_count == expected_requests
-
- def test_keeps_the_answers_of_the_requests_that_succeed(self) -> None:
- late_event = list(CORE_EVENT_CANDIDATES)[-1]
- EventDefinition.objects.create(
- team=self.team, project_id=self.team.project_id, name=late_event, last_seen_at=SEEN_AT
- )
- client = _model_that_believes(
- {"Autocapture": 0.95, CORE_EVENT_CANDIDATES[late_event].label: 0.8}, failing_label="Autocapture"
- )
- requests_per_search = -(-len(CORE_EVENT_CANDIDATES) // GATEWAY_MAX_QUESTIONS)
- with patch(BUILD_CLIENT, return_value=client), patch(CAPTURE) as capture:
- matches = match_core_events(self._search("browser capture")).matches
- match_core_events(self._search("browser capture"))
-
- assert [match.name for match in matches] == [late_event]
- assert capture.call_count == 2
- # A partial answer is not cached, so the second search asks every request again.
- assert client.decide.call_count == 2 * requests_per_search
-
- def test_a_partial_answer_fails_when_every_event_must_be_answered(self) -> None:
- client = _model_that_believes({"Autocapture": 0.95}, failing_label="Autocapture")
- with patch(BUILD_CLIENT, return_value=client), patch(CAPTURE), self.assertRaises(SystemOneRequestFailed):
- likely_core_events(self.team.id, "browser capture", use_cache=False, require_complete=True)
-
- def test_fails_when_every_request_fails(self) -> None:
- client = MagicMock()
- client.decide.side_effect = SystemOneRequestFailed("The ai-gateway was not reached: ReadTimeout")
- with patch(BUILD_CLIENT, return_value=client), patch(CAPTURE), self.assertRaises(SystemOneRequestFailed):
- match_core_events(self._search("browser capture"))
diff --git a/products/posthog_ai/eval_harness/test/test_offline_enrollment.py b/products/posthog_ai/eval_harness/test/test_offline_enrollment.py
index c3b6aec3cca6..5654b62fba45 100644
--- a/products/posthog_ai/eval_harness/test/test_offline_enrollment.py
+++ b/products/posthog_ai/eval_harness/test/test_offline_enrollment.py
@@ -28,8 +28,6 @@ async def test_discovered_suites_match_reporting_enrollment(tmp_path: Path) -> N
with (
patch.object(_BaseEvalRun, "run", capture),
patch("products.posthog_ai.eval_harness.base.build_case_dir", return_value=tmp_path),
- patch("products.posthog_ai.evals.taxonomic_filter.eval_event_match.system_one_configured", return_value=True),
- patch("products.posthog_ai.evals.taxonomic_filter.eval_event_match.EVENT_MATCH_MODEL"),
patch("products.posthog_ai.evals.taxonomic_filter.eval_search_intent.system_one_configured", return_value=True),
patch("products.posthog_ai.evals.taxonomic_filter.eval_search_intent.fetch_search_intent_prompt"),
):
diff --git a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py
index 1712b77cce73..6ee600f521ba 100644
--- a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py
+++ b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py
@@ -467,10 +467,7 @@ async def test_suite_enrolls_every_scorer_in_offline_publishing(
module = importlib.import_module(discovered.fn.__module__)
if domain == "taxonomic_filter":
monkeypatch.setattr(module, "system_one_configured", lambda: True)
- if module_name == "eval_event_match":
- monkeypatch.setattr(module, "EVENT_MATCH_MODEL", MagicMock())
- else:
- monkeypatch.setattr(module, "fetch_search_intent_prompt", MagicMock())
+ monkeypatch.setattr(module, "fetch_search_intent_prompt", MagicMock())
run = AsyncMock()
runner_kind = "OneShot" if suite_kind == SuiteKind.ONE_SHOT else "Sandboxed"
monkeypatch.setattr(module, f"{runner_kind}Eval", run)
diff --git a/products/posthog_ai/evals/offline_config.py b/products/posthog_ai/evals/offline_config.py
index efad9a54bd2d..f341bd32224c 100644
--- a/products/posthog_ai/evals/offline_config.py
+++ b/products/posthog_ai/evals/offline_config.py
@@ -218,9 +218,7 @@ _SCORERS: dict[str, dict[str, OfflineScorer]] = {
"offer_correctness": OfflineScorer(kind="boolean", version=UUID("01a15a29-ea9e-743e-9a21-cec75e5e417e")),
},
"taxonomic_filter": {
- "event_match_found": OfflineScorer(kind="boolean", version=UUID("01a15582-8f96-76a8-a6b0-9ad9c932f12a")),
"exit_code_zero": OfflineScorer(kind="boolean", version=UUID("01a15582-85bb-71aa-b4c8-dc008015ab30")),
- "no_wrong_event_match": OfflineScorer(kind="boolean", version=UUID("01a15582-9ce5-711e-b612-3b0be41c2669")),
"no_wrong_switch": OfflineScorer(kind="boolean", version=UUID("01a15582-b862-766e-b1ee-6664b70a9a93")),
"search_intent_match": OfflineScorer(kind="boolean", version=UUID("01a15582-a5dd-7f46-8681-5f0dd77b22d4")),
"switch_when_needed": OfflineScorer(kind="boolean", version=UUID("01a15582-aef0-748b-8282-c92f901762b5")),
diff --git a/products/posthog_ai/evals/taxonomic_filter/eval_event_match.py b/products/posthog_ai/evals/taxonomic_filter/eval_event_match.py
deleted file mode 100644
index c58a6c96a24f..000000000000
--- a/products/posthog_ai/evals/taxonomic_filter/eval_event_match.py
+++ /dev/null
@@ -1,116 +0,0 @@
-"""Does the decision model find the core event a person describes when no event name matches their search?
-
-The unit tests in `posthog/taxonomic_search_intent/test_event_match.py` feed the matcher canned answers. They
-cover the value rules, the request chunks, the ingested-event check and the cache, and they pass whatever the
-model says. This suite covers the model itself: given a search that matched no event name, does it pick the
-core events the search describes, and nothing else?
-
-The cases are invented from the shapes people type when they do not know an event's name: a description of
-what the event records ("browser capture"), a synonym ("page visit"), a feeling ("angry clicks"). The no-match
-cases are business events that no core event covers, where any suggestion is wrong.
-
-`EventMatchFound` reads as recall and `NoWrongEventMatch` as precision. Precision is the number to protect,
-because a wrong suggestion is worse than none. `MATCH_THRESHOLD` in `posthog/taxonomic_search_intent/event_match.py`
-decides what the picker shows; tune it from this suite.
-
-The suite scores the model's likely events before the ingested-event check, so the demo project's own events
-do not change the result. It shows at most `MAX_MATCHES` events, like the picker.
-
-No CI job runs this suite. Run it by hand, with AI_GATEWAY_URL and AI_GATEWAY_API_KEY set for the decision
-model, and with the harness's own LLM_GATEWAY_ANTHROPIC_API_KEY:
- hogli evals eval_event_match
-Set EVENT_MATCH_PROMPT_VERSION to score an unpublished prompt version before moving its production label.
-Fetching a version needs POSTHOG_PERSONAL_API_KEY with read access to the PostHog project, and the suite fails without it.
-"""
-
-from __future__ import annotations
-
-import os
-import time
-import asyncio
-
-from posthog.llm.system_one_client import system_one_configured
-from posthog.taxonomic_search_intent.event_match import EVENT_MATCH_MODEL, MAX_MATCHES, likely_core_events
-
-from products.posthog_ai.eval_harness.config import BaseEvalCase
-from products.posthog_ai.eval_harness.harness.context import EvalContext
-from products.posthog_ai.eval_harness.harness.requirements import SuiteKind
-from products.posthog_ai.eval_harness.one_shot import OneShotEval
-from products.posthog_ai.evals.offline_config import offline_suite
-from products.posthog_ai.evals.taxonomic_filter.scorers import EVENT_MATCH_KEY, EventMatchFound, NoWrongEventMatch
-
-SUITE_KIND = SuiteKind.ONE_SHOT
-
-
-def _case(name: str, query: str, *acceptable: str) -> BaseEvalCase:
- return BaseEvalCase(name=name, prompt=query, expected={EVENT_MATCH_KEY: {"acceptable": list(acceptable)}})
-
-
-DESCRIBED_CASES = [
- _case("browser_capture", "browser capture", "$autocapture"),
- _case("automatic_clicks", "automatic clicks", "$autocapture"),
- _case("page_visit", "page visit", "$pageview"),
- _case("left_the_page", "left the page", "$pageleave"),
- _case("angry_clicks", "angry clicks", "$rageclick"),
- _case("clicks_that_do_nothing", "clicks that do nothing", "$dead_click"),
- _case("js_error", "js error", "$exception"),
- _case("app_screen", "app screen view", "$screen"),
- _case("page_speed", "page speed", "$web_vitals"),
- _case("llm_call", "llm call", "$ai_generation"),
- _case("copied_text", "copied text", "$copy_autocapture"),
-]
-
-NO_MATCH_CASES = [
- _case("invoice_paid", "invoice paid"),
- _case("subscription_upgraded", "subscription upgraded"),
- _case("file_shared", "file shared"),
-]
-
-
-async def eval_event_match(ctx: EvalContext) -> None:
- # Without a gateway every case errors, which reads as a model regression instead of a setup gap.
- if not system_one_configured():
- raise RuntimeError(
- "eval_event_match needs AI_GATEWAY_URL (https) and AI_GATEWAY_API_KEY to reach the decision model"
- )
- version = os.environ.get("EVENT_MATCH_PROMPT_VERSION")
- model = await asyncio.to_thread(EVENT_MATCH_MODEL.fetch, version=int(version) if version else None)
-
- async def task(case: BaseEvalCase, task_ctx: EvalContext) -> dict:
- if task_ctx.demo_data is None:
- raise RuntimeError("one-shot suites run against the master Hedgebox team")
- started = time.monotonic()
- try:
- # Sync and blocking on the gateway, so keep it off the event loop. No cache: every run asks the model.
- # A failed chunk leaves its events out, which would score as a model miss, so a partial answer is an error.
- likely = await asyncio.to_thread(
- likely_core_events,
- task_ctx.demo_data.master_team_id,
- case.prompt,
- use_cache=False,
- require_complete=True,
- model=model,
- )
- except Exception as error:
- return {"model": model, "suggested": [], "error": f"{type(error).__name__}: {error}"}
- suggested = [match.name for match in likely[:MAX_MATCHES]]
- return {
- "model": model,
- "suggested": suggested,
- "probabilities": {match.name: match.probability for match in likely[:MAX_MATCHES]},
- "latency_ms": round((time.monotonic() - started) * 1000),
- "last_message": f"{case.prompt!r}: {suggested}",
- }
-
- await OneShotEval(
- experiment_name="taxonomic-filter-event-match",
- offline_suite=offline_suite(
- "taxonomic_filter/eval_event_match::eval_event_match",
- metrics=["event_match_found", "no_wrong_event_match"],
- suite_kind=SUITE_KIND,
- ),
- cases=[*DESCRIBED_CASES, *NO_MATCH_CASES],
- scorers=[EventMatchFound(), NoWrongEventMatch()],
- task=task,
- ctx=ctx,
- )
diff --git a/products/posthog_ai/evals/taxonomic_filter/scorers.py b/products/posthog_ai/evals/taxonomic_filter/scorers.py
index 7ff1f2620a5c..976fc641558e 100644
--- a/products/posthog_ai/evals/taxonomic_filter/scorers.py
+++ b/products/posthog_ai/evals/taxonomic_filter/scorers.py
@@ -83,58 +83,3 @@ class NoWrongSwitch(Scorer):
# A switch away from an acceptable open tab is wrong, because the person is already in the right place.
right = spec["active"] not in spec["acceptable"] and intent.get("group_type") in spec["acceptable"]
return Score(name=self._name(), score=1.0 if right else 0.0, metadata={"intent": intent})
-
-
-EVENT_MATCH_KEY = "event_match"
-
-
-def _event_spec(expected: dict | None) -> list[str] | None:
- spec = (expected or {}).get(EVENT_MATCH_KEY)
- return list(spec["acceptable"]) if isinstance(spec, dict) else None
-
-
-def _suggested(output: dict | None) -> list[str]:
- return list((output or {}).get("suggested") or [])
-
-
-class EventMatchFound(Scorer):
- """Recall: for a search that describes a core event, does any suggestion the picker shows name it?"""
-
- def _name(self) -> str:
- return "event_match_found"
-
- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:
- acceptable = _event_spec(expected)
- if not acceptable:
- return Score(name=self._name(), score=None, metadata={"reason": "No core event fits this search"})
- # A failed gateway call says nothing about the model, so it skips rather than scores as a miss.
- if output and output.get("error"):
- return Score(name=self._name(), score=None, metadata={"reason": output["error"]})
- suggested = _suggested(output)
- found = any(name in acceptable for name in suggested)
- return Score(name=self._name(), score=1.0 if found else 0.0, metadata={"suggested": suggested})
-
-
-class NoWrongEventMatch(Scorer):
- """Precision: does every suggestion the picker would show describe the search?
-
- A search that fits no core event must get no suggestion at all, because a wrong event is worse than none.
- """
-
- def _name(self) -> str:
- return "no_wrong_event_match"
-
- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:
- acceptable = _event_spec(expected)
- if acceptable is None:
- return Score(name=self._name(), score=None, metadata={"reason": "No expectation for this case"})
- if output and output.get("error"):
- return Score(name=self._name(), score=None, metadata={"reason": output["error"]})
- suggested = _suggested(output)
- # No suggestion is right for a search that fits no event, and says nothing about precision for one that does.
- if not suggested:
- if acceptable:
- return Score(name=self._name(), score=None, metadata={"reason": "No suggestion shown"})
- return Score(name=self._name(), score=1.0, metadata={"suggested": suggested})
- right = all(name in acceptable for name in suggested)
- return Score(name=self._name(), score=1.0 if right else 0.0, metadata={"suggested": suggested})
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index c82120276775..ba972c8f9209 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -46435,62 +46435,6 @@ export namespace Schemas {
pipelines: IngestionPipelineEnum[];
}
- export interface EventMatch {
- /** The event name to select, such as $autocapture. */
- name: string;
- /** The event's display name, such as Autocapture. */
- display_name: string;
- /** How likely the search means this event, from 0 to 1. */
- probability: number;
- }
-
- /**
- * * `wrong_length` - The search is too short or too long to ask about
- * * `only_values` - Nothing but values is left for the model to read
- * * `unavailable` - The model could not answer at all
- * * `partial` - Some model requests failed, so an event it never asked about might have matched
- * * `nothing_likely` - The model found no core event likely
- * * `not_ingested` - The project never sent the likely events
- * * `matched` - At least one likely event has data
- */
- export type EventMatchOutcomeEnum = typeof EventMatchOutcomeEnum[keyof typeof EventMatchOutcomeEnum];
-
-
- export const EventMatchOutcomeEnum = {
- WrongLength: 'wrong_length',
- OnlyValues: 'only_values',
- Unavailable: 'unavailable',
- Partial: 'partial',
- NothingLikely: 'nothing_likely',
- NotIngested: 'not_ingested',
- Matched: 'matched',
- } as const;
-
- export interface EventMatchRequest {
- /**
- * What the person typed into the events list search box, which matched no event name.
- * @maxLength 200
- */
- query: string;
- }
-
- export interface EventMatchResponse {
- /** PostHog core events the search most likely means, strongest first. Empty when nothing is likely. */
- matches: EventMatch[];
- /** The step that decided the answer, so an empty answer says why it is empty.
- *
- * * `wrong_length` - The search is too short or too long to ask about
- * * `only_values` - Nothing but values is left for the model to read
- * * `unavailable` - The model could not answer at all
- * * `partial` - Some model requests failed, so an event it never asked about might have matched
- * * `nothing_likely` - The model found no core event likely
- * * `not_ingested` - The project never sent the likely events
- * * `matched` - At least one likely event has data */
- outcome: EventMatchOutcomeEnum;
- /** Likely core events this project never sent, strongest first. Only filled when outcome is not_ingested. */
- unsent: EventMatch[];
- }
-
export type EventMatchScope = typeof EventMatchScope[keyof typeof EventMatchScope];