:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(business-knowledge): log why a learning model call failed (#121893)
Alex V7891b2eed6b2
diff --git a/products/business_knowledge/backend/temporal/learning/activities/analyze.py b/products/business_knowledge/backend/temporal/learning/activities/analyze.pyindex 816f4b093937..a99f100af2fd 100644--- a/products/business_knowledge/backend/temporal/learning/activities/analyze.py+++ b/products/business_knowledge/backend/temporal/learning/activities/analyze.py@@ -8,7 +8,7 @@ from django.db import transactionimport structlogfrom langchain_core.messages import HumanMessage, SystemMessagefrom posthoganalytics.ai.langchain.callbacks import CallbackHandler-from pydantic import BaseModel+from pydantic import BaseModel, ValidationErrorfrom temporalio import activityfrom temporalio.common import MetricMeter@@ -183,6 +183,47 @@ def _build_model()+def _provider_error_type(body: object) -> str | None:+ if not isinstance(body, dict):+ return None+ error = body.get("error")+ if not isinstance(error, dict):+ return None+ error_type = error.get("type")+ if not isinstance(error_type, str) or not error_type:+ return None+ return error_type+++def _validation_failures(error: ValidationError, output_model: type[BaseModel]) -> list[str]:+ known_fields = output_model.model_fields.keys()+ failures: list[str] = []+ for detail in error.errors(include_url=False, include_context=False, include_input=False):+ location = detail["loc"]+ # An extra_forbidden location is a key that the model wrote, so only schema field names are logged.+ field = str(location[0]) if location and location[0] in known_fields else "<other>"+ failures.append(f"{field}:{detail['type']}")+ return failures+++def _model_failure_fields(error: Exception, output_model: type[BaseModel]) -> dict[str, str | int | list[str]]:+ fields: dict[str, str | int | list[str]] = {"error_type": type(error).__name__}+ if error.__cause__ is not None:+ fields["cause_type"] = type(error.__cause__).__name__+ if isinstance(error, ValidationError):+ fields["validation_failures"] = _validation_failures(error, output_model)+ status_code = getattr(error, "status_code", None)+ if isinstance(status_code, int):+ fields["status_code"] = status_code+ request_id = getattr(error, "request_id", None)+ if isinstance(request_id, str) and request_id:+ fields["request_id"] = request_id+ provider_error_type = _provider_error_type(getattr(error, "body", None))+ if provider_error_type is not None:+ fields["provider_error_type"] = provider_error_type+ return fields++def _invoke_structured_model(team: Team,user: User,@@ -209,7 +250,15 @@ def _invoke_structured_model(result = model.invoke(messages)else:result = model.invoke(messages, config={"callbacks": [callback]})- except Exception:+ except Exception as error:+ logger.error( # noqa: TRY400 — exception() records the provider message, which can quote the support reply+ "business_knowledge.learning.model_failed",+ stage=stage,+ team_id=team.id,+ trace_id=trace_id,+ **_model_failure_fields(error, output_model),+ )+ # The provider message can quote the support reply, and workflow history stores __cause__.raise LearningAnalysisError(f"{stage}_model_failed") from Noneif not isinstance(result, output_model):raise LearningAnalysisError(f"invalid_{stage}_output") from Nonediff --git a/products/business_knowledge/backend/tests/test_learning_analyzer.py b/products/business_knowledge/backend/tests/test_learning_analyzer.pyindex 06ff84c85840..b2ac5998eb8b 100644--- a/products/business_knowledge/backend/tests/test_learning_analyzer.py+++ b/products/business_knowledge/backend/tests/test_learning_analyzer.py@@ -1,3 +1,4 @@+from collections.abc import Callablefrom datetime import UTC, datetime, timedeltafrom typing import castfrom uuid import UUID@@ -5,8 +6,12 @@ from uuid import UUIDimport pytestfrom unittest.mock import MagicMock, patch+import httpx+import anthropicfrom asgiref.sync import sync_to_asyncfrom posthoganalytics.ai.langchain.callbacks import CallbackHandler+from pydantic import ValidationError+from structlog.testing import capture_logsfrom temporalio.common import MetricMeterfrom temporalio.testing import ActivityEnvironment@@ -160,6 +165,35 @@ def _existing_result(source: KnowledgeSource, content: str) -> logic.KnowledgeSe)+def _provider_status_error() -> Exception:+ request = httpx.Request("POST", "https://api.anthropic.com/v1/messages")+ return anthropic.AuthenticationError(+ "sensitive model output",+ response=httpx.Response(401, request=request, headers={"request-id": "req_test"}),+ body={"type": "error", "error": {"type": "authentication_error", "message": "sensitive model output"}},+ )+++def _provider_connection_error() -> Exception:+ error = anthropic.APIConnectionError(request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"))+ error.__cause__ = httpx.ProxyError("sensitive model output")+ return error+++def _extraction_validation_error() -> Exception:+ payload = {+ **_extraction().model_dump(),+ "canonical_answer": "",+ "rejection_code": "sensitive model output",+ "sensitive model output": True,+ }+ try:+ ExtractedKnowledge.model_validate(payload)+ except ValidationError as error:+ return error+ raise AssertionError("payload must fail validation")++def _embedding() -> EmbeddingResponse:return EmbeddingResponse(embedding=[0.1, 0.2], tokens_used=4, did_truncate=False)@@ -480,7 +514,44 @@ class TestLearningAnalyzer:search.assert_not_called()publish.assert_not_called()- def test_model_error_marks_run_failed_without_capturing_sensitive_error(self, team: Team) -> None:+ @pytest.mark.parametrize(+ "error_factory,expected_fields",+ [+ pytest.param(+ _provider_status_error,+ {+ "error_type": "AuthenticationError",+ "status_code": 401,+ "request_id": "req_test",+ "provider_error_type": "authentication_error",+ },+ id="provider_status",+ ),+ pytest.param(+ _provider_connection_error,+ {"error_type": "APIConnectionError", "cause_type": "ProxyError"},+ id="provider_connection",+ ),+ pytest.param(+ _extraction_validation_error,+ {+ "error_type": "ValidationError",+ "validation_failures": [+ "canonical_answer:string_too_short",+ "rejection_code:literal_error",+ "<other>:extra_forbidden",+ ],+ },+ id="output_validation",+ ),+ ],+ )+ def test_model_error_marks_run_failed_without_capturing_sensitive_error(+ self,+ team: Team,+ error_factory: Callable[[], Exception],+ expected_fields: dict[str, object],+ ) -> None:run, input = _setup_sync(team)provider = _Provider(EvidenceBundle(replies=("Refunds are available within 30 days.",)))model = MagicMock()@@ -490,7 +561,7 @@ class TestLearningAnalyzer:def _raise_sensitive_error(messages: object, config: dict[str, list[CallbackHandler]]) -> None:callback = config["callbacks"][0]- error = ValueError("sensitive model output")+ error = error_factory()callback.on_llm_error(error, run_id=UUID(int=1))callback.on_chain_error(error, run_id=UUID(int=2))raise error@@ -501,13 +572,26 @@ class TestLearningAnalyzer:patch(f"{_MODULE}.get_learning_provider", return_value=provider),patch(f"{_MODULE}._build_model", return_value=model),patch("products.business_knowledge.backend.llm_telemetry.posthoganalytics", analytics),- pytest.raises(LearningAnalysisError, match="extraction_model_failed"),+ capture_logs() as logs,+ pytest.raises(LearningAnalysisError, match="extraction_model_failed") as raised,):analyze_learning_evidence(input)+ assert raised.value.__cause__ is Nonerun.refresh_from_db()assert run.status == "failed"assert run.error == "extraction_model_failed"+ failed = [entry for entry in logs if entry["event"] == "business_knowledge.learning.model_failed"]+ assert failed == [+ {+ "event": "business_knowledge.learning.model_failed",+ "log_level": "error",+ "stage": "extraction",+ "team_id": team.id,+ "trace_id": str(run.id),+ **expected_fields,+ }+ ]model.with_structured_output.assert_called_once_with(ExtractedKnowledge,method="json_schema",