Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(business-knowledge): log why a learning model call failed (#121893)

Alex V7891b2eed6b2
diff --git a/products/business_knowledge/backend/temporal/learning/activities/analyze.py b/products/business_knowledge/backend/temporal/learning/activities/analyze.py
index 816f4b093937..a99f100af2fd 100644
--- a/products/business_knowledge/backend/temporal/learning/activities/analyze.py
+++ b/products/business_knowledge/backend/temporal/learning/activities/analyze.py
@@ -8,7 +8,7 @@ from django.db import transaction
import structlog
from langchain_core.messages import HumanMessage, SystemMessage
from posthoganalytics.ai.langchain.callbacks import CallbackHandler
-from pydantic import BaseModel
+from pydantic import BaseModel, ValidationError
from temporalio import activity
from temporalio.common import MetricMeter
@@ -183,6 +183,47 @@ def _build_model(
)
+def _provider_error_type(body: object) -> str | None:
+ if not isinstance(body, dict):
+ return None
+ error = body.get("error")
+ if not isinstance(error, dict):
+ return None
+ error_type = error.get("type")
+ if not isinstance(error_type, str) or not error_type:
+ return None
+ return error_type
+
+
+def _validation_failures(error: ValidationError, output_model: type[BaseModel]) -> list[str]:
+ known_fields = output_model.model_fields.keys()
+ failures: list[str] = []
+ for detail in error.errors(include_url=False, include_context=False, include_input=False):
+ location = detail["loc"]
+ # An extra_forbidden location is a key that the model wrote, so only schema field names are logged.
+ field = str(location[0]) if location and location[0] in known_fields else "<other>"
+ failures.append(f"{field}:{detail['type']}")
+ return failures
+
+
+def _model_failure_fields(error: Exception, output_model: type[BaseModel]) -> dict[str, str | int | list[str]]:
+ fields: dict[str, str | int | list[str]] = {"error_type": type(error).__name__}
+ if error.__cause__ is not None:
+ fields["cause_type"] = type(error.__cause__).__name__
+ if isinstance(error, ValidationError):
+ fields["validation_failures"] = _validation_failures(error, output_model)
+ status_code = getattr(error, "status_code", None)
+ if isinstance(status_code, int):
+ fields["status_code"] = status_code
+ request_id = getattr(error, "request_id", None)
+ if isinstance(request_id, str) and request_id:
+ fields["request_id"] = request_id
+ provider_error_type = _provider_error_type(getattr(error, "body", None))
+ if provider_error_type is not None:
+ fields["provider_error_type"] = provider_error_type
+ return fields
+
+
def _invoke_structured_model(
team: Team,
user: User,
@@ -209,7 +250,15 @@ def _invoke_structured_model(
result = model.invoke(messages)
else:
result = model.invoke(messages, config={"callbacks": [callback]})
- except Exception:
+ except Exception as error:
+ logger.error( # noqa: TRY400 — exception() records the provider message, which can quote the support reply
+ "business_knowledge.learning.model_failed",
+ stage=stage,
+ team_id=team.id,
+ trace_id=trace_id,
+ **_model_failure_fields(error, output_model),
+ )
+ # The provider message can quote the support reply, and workflow history stores __cause__.
raise LearningAnalysisError(f"{stage}_model_failed") from None
if not isinstance(result, output_model):
raise LearningAnalysisError(f"invalid_{stage}_output") from None
diff --git a/products/business_knowledge/backend/tests/test_learning_analyzer.py b/products/business_knowledge/backend/tests/test_learning_analyzer.py
index 06ff84c85840..b2ac5998eb8b 100644
--- a/products/business_knowledge/backend/tests/test_learning_analyzer.py
+++ b/products/business_knowledge/backend/tests/test_learning_analyzer.py
@@ -1,3 +1,4 @@
+from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from typing import cast
from uuid import UUID
@@ -5,8 +6,12 @@ from uuid import UUID
import pytest
from unittest.mock import MagicMock, patch
+import httpx
+import anthropic
from asgiref.sync import sync_to_async
from posthoganalytics.ai.langchain.callbacks import CallbackHandler
+from pydantic import ValidationError
+from structlog.testing import capture_logs
from temporalio.common import MetricMeter
from temporalio.testing import ActivityEnvironment
@@ -160,6 +165,35 @@ def _existing_result(source: KnowledgeSource, content: str) -> logic.KnowledgeSe
)
+def _provider_status_error() -> Exception:
+ request = httpx.Request("POST", "https://api.anthropic.com/v1/messages")
+ return anthropic.AuthenticationError(
+ "sensitive model output",
+ response=httpx.Response(401, request=request, headers={"request-id": "req_test"}),
+ body={"type": "error", "error": {"type": "authentication_error", "message": "sensitive model output"}},
+ )
+
+
+def _provider_connection_error() -> Exception:
+ error = anthropic.APIConnectionError(request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"))
+ error.__cause__ = httpx.ProxyError("sensitive model output")
+ return error
+
+
+def _extraction_validation_error() -> Exception:
+ payload = {
+ **_extraction().model_dump(),
+ "canonical_answer": "",
+ "rejection_code": "sensitive model output",
+ "sensitive model output": True,
+ }
+ try:
+ ExtractedKnowledge.model_validate(payload)
+ except ValidationError as error:
+ return error
+ raise AssertionError("payload must fail validation")
+
+
def _embedding() -> EmbeddingResponse:
return EmbeddingResponse(embedding=[0.1, 0.2], tokens_used=4, did_truncate=False)
@@ -480,7 +514,44 @@ class TestLearningAnalyzer:
search.assert_not_called()
publish.assert_not_called()
- def test_model_error_marks_run_failed_without_capturing_sensitive_error(self, team: Team) -> None:
+ @pytest.mark.parametrize(
+ "error_factory,expected_fields",
+ [
+ pytest.param(
+ _provider_status_error,
+ {
+ "error_type": "AuthenticationError",
+ "status_code": 401,
+ "request_id": "req_test",
+ "provider_error_type": "authentication_error",
+ },
+ id="provider_status",
+ ),
+ pytest.param(
+ _provider_connection_error,
+ {"error_type": "APIConnectionError", "cause_type": "ProxyError"},
+ id="provider_connection",
+ ),
+ pytest.param(
+ _extraction_validation_error,
+ {
+ "error_type": "ValidationError",
+ "validation_failures": [
+ "canonical_answer:string_too_short",
+ "rejection_code:literal_error",
+ "<other>:extra_forbidden",
+ ],
+ },
+ id="output_validation",
+ ),
+ ],
+ )
+ def test_model_error_marks_run_failed_without_capturing_sensitive_error(
+ self,
+ team: Team,
+ error_factory: Callable[[], Exception],
+ expected_fields: dict[str, object],
+ ) -> None:
run, input = _setup_sync(team)
provider = _Provider(EvidenceBundle(replies=("Refunds are available within 30 days.",)))
model = MagicMock()
@@ -490,7 +561,7 @@ class TestLearningAnalyzer:
def _raise_sensitive_error(messages: object, config: dict[str, list[CallbackHandler]]) -> None:
callback = config["callbacks"][0]
- error = ValueError("sensitive model output")
+ error = error_factory()
callback.on_llm_error(error, run_id=UUID(int=1))
callback.on_chain_error(error, run_id=UUID(int=2))
raise error
@@ -501,13 +572,26 @@ class TestLearningAnalyzer:
patch(f"{_MODULE}.get_learning_provider", return_value=provider),
patch(f"{_MODULE}._build_model", return_value=model),
patch("products.business_knowledge.backend.llm_telemetry.posthoganalytics", analytics),
- pytest.raises(LearningAnalysisError, match="extraction_model_failed"),
+ capture_logs() as logs,
+ pytest.raises(LearningAnalysisError, match="extraction_model_failed") as raised,
):
analyze_learning_evidence(input)
+ assert raised.value.__cause__ is None
run.refresh_from_db()
assert run.status == "failed"
assert run.error == "extraction_model_failed"
+ failed = [entry for entry in logs if entry["event"] == "business_knowledge.learning.model_failed"]
+ assert failed == [
+ {
+ "event": "business_knowledge.learning.model_failed",
+ "log_level": "error",
+ "stage": "extraction",
+ "team_id": team.id,
+ "trace_id": str(run.id),
+ **expected_fields,
+ }
+ ]
model.with_structured_output.assert_called_once_with(
ExtractedKnowledge,
method="json_schema",