Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

10.9 KB
Raw Blame
"""Classify MCP tool-call failure messages into a fixed set of causes.

Staff-tool support code for ``classify_mcp_failures``: normalizes raw error messages into
fingerprints (pure, no LLM) and classifies fingerprints in batches via an LLM call, mirroring the
pattern in ``intent_generation.py``.
"""

import re
from enum import StrEnum

import openai
import structlog
from openai import OpenAI
from pydantic import BaseModel, ValidationError

from posthog.clickhouse.query_tagging import Feature, Product, tags_context
from posthog.llm.gateway_client import build_openai_client, resolve_ai_gateway_config
from posthog.models.team.team import Team

logger = structlog.get_logger(__name__)


class FailureClassificationUnavailable(RuntimeError):
    pass


FAILURE_CLASS_MODEL = "gpt-5.6-luna"
CLASSIFY_BATCH_SIZE = 25
FINGERPRINT_MAX_CHARS = 200
_QUOTED_VALUE_MAX_CHARS = 80


class FailureClass(StrEnum):
    INPUT_SCHEMA_MISMATCH = "input_schema_mismatch"
    OUTPUT_SCHEMA_VIOLATION = "output_schema_violation"
    UNKNOWN_TOOL = "unknown_tool"
    GATED_TOOL = "gated_tool"
    TIMEOUT = "timeout"
    UPSTREAM_5XX = "upstream_5xx"
    TRANSPORT_FAILURE = "transport_failure"
    RESOURCE_NOT_FOUND = "resource_not_found"
    EMPTY_RESULT_AS_ERROR = "empty_result_as_error"
    STATE_CONFLICT = "state_conflict"
    MISSING_PRECONDITION = "missing_precondition"
    AUTH_OR_PERMISSION = "auth_or_permission"
    ENTITLEMENT_GAP = "entitlement_gap"
    RATE_LIMITED = "rate_limited"
    INTERNAL_ERROR = "internal_error"


FAILURE_CLASSES: tuple[str, ...] = tuple(cls.value for cls in FailureClass)

_UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.IGNORECASE)
_EMAIL_RE = re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b")
_URL_RE = re.compile(r"\bhttps?://\S+", re.IGNORECASE)
_ISO_DATE_RE = re.compile(
    r"\b\d{4}-\d{2}-\d{2}(?:[t ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:z|[+-]\d{2}:?\d{2})?)?\b", re.IGNORECASE
)
_DOUBLE_QUOTED_RE = re.compile(rf'"[^"]{{0,{_QUOTED_VALUE_MAX_CHARS}}}"')
_SINGLE_QUOTED_RE = re.compile(rf"'[^']{{0,{_QUOTED_VALUE_MAX_CHARS}}}'")
_NUMBER_RE = re.compile(r"\d+")
_WHITESPACE_RE = re.compile(r"\s+")


def normalize_fingerprint(msg: str) -> str:
    """Collapse a raw error message into a stable fingerprint for grouping.

    Strips values that vary per-occurrence (IDs, timestamps, quoted literals, numbers) but leaves
    the message's language and wording untouched — a translated message must stay a distinct
    fingerprint from its English twin, since fingerprinting is grouping, not classification.
    """
    normalized = msg.lower()
    normalized = _URL_RE.sub(" ", normalized)
    normalized = _EMAIL_RE.sub(" ", normalized)
    normalized = _UUID_RE.sub(" ", normalized)
    normalized = _ISO_DATE_RE.sub(" ", normalized)
    normalized = _DOUBLE_QUOTED_RE.sub(" ", normalized)
    normalized = _SINGLE_QUOTED_RE.sub(" ", normalized)
    normalized = _NUMBER_RE.sub(" ", normalized)
    normalized = _WHITESPACE_RE.sub(" ", normalized).strip()
    return normalized[:FINGERPRINT_MAX_CHARS]


SYSTEM_PROMPT = f"""You classify MCP tool-call failure messages into exactly one of these {len(FAILURE_CLASSES)} \
classes:

- input_schema_mismatch: caller's arguments rejected against the tool's declared input contract \
— "Invalid arguments: expected array, received undefined", "The <field> field is required"
- output_schema_violation: the server's own response violates its declared output schema — \
"Output validation error: invalid structured content"
- unknown_tool: a tool that doesn't exist was called — "Unknown tool: <name>", JSON-RPC -32601
- gated_tool: the tool exists but this connection cannot see it, so the message names a \
successor or says the tool is not enabled — "is retired on this PostHog connection. Use <name> \
instead", "exists, but it is not enabled on this PostHog connection"
- timeout: the operation ran out of time — "timed out after 45s"
- upstream_5xx: a dependency behind the server failed — "API request failed: 503"
- transport_failure: network-level failure — "fetch failed", "connection reset"
- resource_not_found: the referenced entity doesn't exist — "Resource does not exist — verify the ID"
- empty_result_as_error: a search legitimately found nothing but was surfaced as an error — "No \
matching records found; adjust your query"
- state_conflict: entity exists but its current state forbids the operation — "already exists", \
"not in draft status"
- missing_precondition: one-time setup never done — "no account connected", "sender email must \
be verified"
- auth_or_permission: identity/credential/scope problem — 401/403, "insufficient scopes", \
"read-only key"
- entitlement_gap: authentication fine, the caller's plan doesn't include this — "not available \
on the free tier", "upgrade your plan"
- rate_limited: throttled on volume/speed — 429, "retry after N seconds"
- internal_error: the message carries no diagnosable cause — "tool failed", "[object Object]", \
bare stack fragments

Boundary rules:
1. Trust the message over the JSON-RPC code: "-32602: Tool X not found" is unknown_tool; \
"-32602: Output validation error" is output_schema_violation.
2. 429 + "upgrade your plan" is entitlement_gap, not rate_limited. "Retry after N seconds" is \
rate_limited.
3. 403 + "requires premium/subscription" is entitlement_gap, not auth_or_permission.
4. missing_precondition vs state_conflict: would retrying tomorrow with no setup action still \
fail? Yes = missing_precondition; depends on a live entity's current state = state_conflict.
5. resource_not_found vs empty_result_as_error: lookup by identifier that missed = \
resource_not_found; search/filter with zero rows = empty_result_as_error.
6. "The X field is required" at call time is input_schema_mismatch unless the requirement is \
business-conditional.
7. Language is irrelevant: a translated message classifies the same as its English twin.
8. A field's NAME never picks the class ("timeout seconds field must not be greater than 25" is \
input_schema_mismatch, not timeout).
9. gated_tool vs unknown_tool: the message says the tool exists but is not enabled, or names a \
replacement = gated_tool. It says the name is unknown = unknown_tool.
10. If the message carries an upsell anywhere ("here are your options", plan limits), \
entitlement_gap wins. If genuinely no cause is recoverable, internal_error — never guess.

Each input line is numbered. Return exactly one classification per input line, in the same order, \
as JSON of exactly this shape:
{{"classifications": [{{"line_number": 1, "failure_class": "<one of the {len(FAILURE_CLASSES)} class \
names>"}}, ...]}}"""

# The LLM reads the raw representative message, never the normalized fingerprint: normalization
# strips numbers, so the status and JSON-RPC codes the boundary rules pivot on (429, 403, -32601)
# would be gone before classification. The fingerprint is only the grouping/cache key.
MESSAGE_MAX_CHARS = 400


class FingerprintClassification(BaseModel):
    line_number: int
    failure_class: FailureClass


class ClassificationBatch(BaseModel):
    classifications: list[FingerprintClassification]


def _build_batch_prompt(messages: list[str]) -> str:
    numbered = "\n".join(f"{i + 1}. {message[:MESSAGE_MAX_CHARS]}" for i, message in enumerate(messages))
    return f"Failure messages:\n{numbered}"


def _call_llm(client: OpenAI, messages: list[str], team: Team) -> ClassificationBatch:
    response = client.chat.completions.create(
        model=FAILURE_CLASS_MODEL,
        timeout=30,
        messages=[
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user", "content": _build_batch_prompt(messages)},
        ],
        response_format={"type": "json_object"},
        user=f"team/{team.id}/mcp-analytics-failure-classifier",
    )
    content = response.choices[0].message.content if response.choices else None
    if not content:
        raise ValueError("LLM returned an empty classification response")
    return ClassificationBatch.model_validate_json(content)


def _classify_batch(client: OpenAI, batch: list[tuple[str, str]], team: Team) -> dict[str, str]:
    """Classify one (fingerprint, message) batch, retrying once on a malformed or invalid-class response."""
    messages = [message for _, message in batch]
    for attempt in range(2):
        try:
            parsed = _call_llm(client, messages, team)
        except (openai.OpenAIError, ValidationError, ValueError):
            if attempt == 1:
                break
            continue
        # Valid JSON with missing/duplicate/out-of-range line numbers is as malformed as a
        # schema violation: it must consume the retry, not silently orphan inputs.
        returned_lines = sorted(item.line_number for item in parsed.classifications)
        if returned_lines != list(range(1, len(batch) + 1)):
            if attempt == 1:
                break
            continue
        by_line = {item.line_number: item.failure_class.value for item in parsed.classifications}
        return {fingerprint: by_line[i + 1] for i, (fingerprint, _) in enumerate(batch)}
    logger.warning("mcp_analytics.failure_classifier.batch_failed", team_id=team.id, batch_size=len(batch))
    return dict.fromkeys((fingerprint for fingerprint, _ in batch), FailureClass.INTERNAL_ERROR.value)


def classify_fingerprints(messages_by_fingerprint: dict[str, str], team: Team) -> dict[str, str]:
    """Classify each fingerprint into one of the 14 failure classes via batched LLM calls.

    ``messages_by_fingerprint`` maps each fingerprint to its representative raw message — the LLM
    reads the message (see the note above ``MESSAGE_MAX_CHARS``), the fingerprint keys the result.
    One call per batch of ``CLASSIFY_BATCH_SIZE``. A batch that fails validation is retried once;
    if it still fails, every fingerprint in that batch is marked ``internal_error`` rather than
    left unclassified.
    """
    if not messages_by_fingerprint:
        return {}

    # Failure messages are customer-authored text, so shipping them to the LLM needs the
    # organization's AI data processing consent, same as intent generation.
    if not team.organization.is_ai_data_processing_approved:
        raise FailureClassificationUnavailable("AI data processing is not approved for this organization")

    if resolve_ai_gateway_config() is None:
        raise FailureClassificationUnavailable("AI gateway is not configured")
    client = build_openai_client(
        "django",
        ai_product="mcp_analytics",
        properties={"ai_feature": "failure-classification"},
        distinct_id=f"team/{team.id}/mcp-analytics-failure-classifier",
    )
    items = list(messages_by_fingerprint.items())
    result: dict[str, str] = {}
    with tags_context(
        product=Product.MCP_ANALYTICS, feature=Feature.QUERY, team_id=team.id, name="mcp_analytics_failure_classifier"
    ):
        for start in range(0, len(items), CLASSIFY_BATCH_SIZE):
            result.update(_classify_batch(client, items[start : start + CLASSIFY_BATCH_SIZE], team))
    return result