:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix: temporal bk (#124653)
Alex V91231f6f5b6e
diff --git a/products/business_knowledge/backend/temporal/learning/activities/analyze.py b/products/business_knowledge/backend/temporal/learning/activities/analyze.pyindex 8bbdc75174f0..7d0f425d84f5 100644--- a/products/business_knowledge/backend/temporal/learning/activities/analyze.py+++ b/products/business_knowledge/backend/temporal/learning/activities/analyze.py@@ -116,6 +116,11 @@ class LearningOutputTruncated(LearningAnalysisError):super().__init__(f"{stage}_output_truncated")+class LearningConflictingOutput(LearningAnalysisError):+ def __init__(self, stage: str) -> None:+ super().__init__(f"{stage}_conflicting_output")++class _SupersessionUnavailable(Exception):"""The conflicting source is not one this job may replace, so the candidate must not be published."""@@ -202,12 +207,19 @@ def _log_model_failure(team: Team, stage: str, trace_id: str, error: Exception,)-def _parse_tool_result(content: object, output_model: type[ModelOutput], stage: str) -> ModelOutput:+def _parse_tool_result(+ content: object, output_model: type[ModelOutput], stage: str, *, first_call_wins: bool+) -> ModelOutput:tool_name = output_model.__name__blocks = content if isinstance(content, list) else []tool_results = [block for block in blocks if isinstance(block, ToolUseBlock) and block.name == tool_name]- if len(tool_results) != 1:+ if not tool_results:raise LearningAnalysisError(f"invalid_{stage}_output")+ if len(tool_results) > 1:+ # The request disables parallel tool use. Several calls can still arrive.+ logger.warning("business_knowledge.learning.multiple_tool_calls", stage=stage, count=len(tool_results))+ if not first_call_wins and any(block.input != tool_results[0].input for block in tool_results[1:]):+ raise LearningConflictingOutput(stage)return output_model.model_validate(tool_results[0].input)@@ -220,6 +232,7 @@ def _invoke_structured_model(payload: dict[str, object],output_model: type[ModelOutput],max_tokens: int = LEARNING_MAX_TOKENS,+ first_call_wins: bool = False,) -> ModelOutput:try:client = build_ai_gateway_anthropic_client(@@ -250,11 +263,13 @@ def _invoke_structured_model("input_schema": output_model.model_json_schema(),}],- tool_choice={"type": "tool", "name": tool_name},+ tool_choice={"type": "tool", "name": tool_name, "disable_parallel_tool_use": True},extra_headers={"X-PostHog-Privacy-Mode": "true"},)try:- return _parse_tool_result(response.content, output_model, stage)+ return _parse_tool_result(response.content, output_model, stage, first_call_wins=first_call_wins)+ except LearningConflictingOutput:+ raiseexcept (LearningAnalysisError, ValidationError) as error:if response.stop_reason == "max_tokens":logger.warning(@@ -292,6 +307,8 @@ def _extract_candidate(team: Team, bundle: EvidenceBundle, trace_id: str) -> Extsystem_prompt=_EXTRACTION_SYSTEM_PROMPT,payload={"public_human_replies": list(bundle.replies)},output_model=ExtractedKnowledge,+ # Every later gate checks the candidate this stage returns, so any one of several candidates is safe to keep.+ first_call_wins=True,)@@ -552,6 +569,7 @@ def _log_analysis_failure(run: KnowledgeLearningRun, error: Exception) -> None:provider=run.provider,evidence_key=run.evidence_key,error_type=type(error).__name__,+ error_code=error.code if isinstance(error, LearningAnalysisError) else None,)@@ -736,7 +754,11 @@ def _analyze(run: KnowledgeLearningRun, input: AnalyzeLearningEvidenceInput) ->return _finish_without_knowledge(run, rejection_code=rejection_code)generated_text = f"{extracted.canonical_topic}\n{extracted.canonical_answer}"- if _model_pii_verdict(run.team, extracted, trace_id).verdict != "safe":+ try:+ pii_verdict = _model_pii_verdict(run.team, extracted, trace_id)+ except LearningConflictingOutput:+ return _finish_without_knowledge(run, rejection_code="pii")+ if pii_verdict.verdict != "safe":return _finish_without_knowledge(run, rejection_code="pii")try:@@ -744,10 +766,16 @@ def _analyze(run: KnowledgeLearningRun, input: AnalyzeLearningEvidenceInput) ->except Exception:raise LearningAnalysisError("knowledge_search_failed") from Noneretrieved = _render_search_context(results)- decision = _promotion_decision(run.team, bundle, extracted, retrieved, trace_id)+ try:+ decision = _promotion_decision(run.team, bundle, extracted, retrieved, trace_id)+ except LearningConflictingOutput:+ return _finish_without_knowledge(run, rejection_code="low_confidence")conflicting = _conflicting_search_result(decision, results, len(retrieved))if conflicting is not None:- confirmation = _confirm_contradiction(run.team, extracted, conflicting, trace_id)+ try:+ confirmation = _confirm_contradiction(run.team, extracted, conflicting, trace_id)+ except LearningConflictingOutput:+ return _finish_without_knowledge(run, rejection_code="already_known")if _is_confirmed_contradiction(confirmation):if not conflicting.is_generated:# Learning may retire what it wrote. A source a person wrote stays until a person changes it.diff --git a/products/business_knowledge/backend/temporal/learning/schemas.py b/products/business_knowledge/backend/temporal/learning/schemas.pyindex 8a73ba56efc5..635e5ed27206 100644--- a/products/business_knowledge/backend/temporal/learning/schemas.py+++ b/products/business_knowledge/backend/temporal/learning/schemas.py@@ -1,7 +1,8 @@from dataclasses import dataclass-from typing import Literal+from typing import Literal, get_args-from pydantic import BaseModel, ConfigDict, Field, StrictBool+import structlog+from pydantic import BaseModel, ConfigDict, Field, StrictBool, field_validatorfrom posthog.dataclasses import frozen@@ -35,6 +36,17 @@ PromotionRejectionCode = Literal["stale_candidate",]+logger = structlog.get_logger(__name__)+++def _known_rejection_code(value: object, allowed: tuple[str, ...], stage: str) -> object:+ # Tool input does not enforce the schema enum, and temperature 0 returns the same invented code on every retry.+ # A rejection stops publication, so mapping an unknown code to one fails closed and lets the run complete.+ if value in allowed:+ return value+ logger.warning("business_knowledge.learning.unknown_rejection_code", stage=stage)+ return "not_useful"+@dataclass(frozen=True)class AnalyzeLearningEvidenceInput:@@ -87,6 +99,11 @@ class ExtractedKnowledge(BaseModel):confidence: float = Field(ge=0, le=1)rejection_code: ExtractionRejectionCode+ @field_validator("rejection_code", mode="before")+ @classmethod+ def _unknown_rejection_code_rejects(cls, value: object) -> object:+ return _known_rejection_code(value, get_args(ExtractionRejectionCode), "extraction")+class PiiVerdict(BaseModel):model_config = ConfigDict(extra="forbid", strict=True)@@ -106,6 +123,11 @@ class PromotionDecision(BaseModel):confidence: float = Field(ge=0, le=1)rejection_code: PromotionRejectionCode+ @field_validator("rejection_code", mode="before")+ @classmethod+ def _unknown_rejection_code_rejects(cls, value: object) -> object:+ return _known_rejection_code(value, get_args(PromotionRejectionCode), "promotion")+class ContradictionVerdict(BaseModel):model_config = ConfigDict(extra="forbid", strict=True)diff --git a/products/business_knowledge/backend/tests/test_learning_analyzer.py b/products/business_knowledge/backend/tests/test_learning_analyzer.pyindex 281af3fcce71..8aece4038ff3 100644--- a/products/business_knowledge/backend/tests/test_learning_analyzer.py+++ b/products/business_knowledge/backend/tests/test_learning_analyzer.py@@ -180,9 +180,12 @@ def _provider_connection_error() -> Exception:return error-def _tool_response(name: str, payload: dict[str, object]) -> MagicMock:+def _tool_response(name: str, *payloads: dict[str, object]) -> MagicMock:response = MagicMock()- response.content = [ToolUseBlock(id="toolu_test", name=name, input=payload, type="tool_use")]+ response.content = [+ ToolUseBlock(id=f"toolu_{index}", name=name, input=payload, type="tool_use")+ for index, payload in enumerate(payloads)+ ]return response@@ -319,6 +322,7 @@ class TestLearningAnalyzer:"unsupported",),({"confidence": 0.4, "rejection_code": "low_confidence"}, "low_confidence"),+ ({"rejection_code": "contradiction"}, "not_useful"),],)def test_extraction_rejections_never_search_or_publish(@@ -348,17 +352,21 @@ class TestLearningAnalyzer:publish.assert_not_called()@pytest.mark.parametrize(- "pii_result",- [PiiVerdict(verdict="unsafe"), PiiVerdict(verdict="uncertain")],+ "pii_results",+ [+ pytest.param([PiiVerdict(verdict="unsafe")], id="unsafe"),+ pytest.param([PiiVerdict(verdict="uncertain")], id="uncertain"),+ pytest.param([PiiVerdict(verdict="safe"), PiiVerdict(verdict="unsafe")], id="safe_then_unsafe"),+ ],)- def test_model_pii_rejection_is_fail_closed(self, team: Team, pii_result: PiiVerdict) -> None:+ def test_model_pii_rejection_is_fail_closed(self, team: Team, pii_results: list[PiiVerdict]) -> None:run, input = _setup_sync(team)provider = _Provider(EvidenceBundle(replies=("Use the standard contact process.",)))client, entered = _gateway_client()extraction = _extraction(canonical_topic="Contact policy", canonical_answer="Contact Taylor for help.")entered.messages.create.side_effect = [_tool_response("ExtractedKnowledge", extraction.model_dump()),- _tool_response("PiiVerdict", pii_result.model_dump()),+ _tool_response("PiiVerdict", *(verdict.model_dump() for verdict in pii_results)),]with (@@ -390,6 +398,7 @@ class TestLearningAnalyzer:for call in entered.messages.create.call_args_list:assert call.kwargs["extra_headers"] == {"X-PostHog-Privacy-Mode": "true"}assert call.kwargs["tool_choice"]["type"] == "tool"+ assert call.kwargs["tool_choice"]["disable_parallel_tool_use"] is Trueembed.assert_not_called()search.assert_not_called()publish.assert_not_called()@@ -448,7 +457,16 @@ class TestLearningAnalyzer:assert search.call_args.kwargs["query_embedding"] == [0.1, 0.2]publish.assert_not_called()- def test_promotion_rechecks_evidence_support_before_publication(self, team: Team) -> None:+ @pytest.mark.parametrize(+ "overrides,expected_code",+ [+ ({"supported_by_public_human_resolution": False, "rejection_code": "unsupported"}, "unsupported"),+ ({"rejection_code": "contradiction"}, "not_useful"),+ ],+ )+ def test_promotion_rejections_block_publication(+ self, team: Team, overrides: dict[str, object], expected_code: str+ ) -> None:run, input = _setup_sync(team)provider = _Provider(EvidenceBundle(replies=("Refunds are available within 30 days.",)))@@ -456,11 +474,7 @@ class TestLearningAnalyzer:patch(f"{_MODULE}.get_learning_provider", return_value=provider),patch(f"{_MODULE}._invoke_structured_model",- side_effect=[- _extraction(),- PiiVerdict(verdict="safe"),- _promotion(supported_by_public_human_resolution=False, rejection_code="unsupported"),- ],+ side_effect=[_extraction(), PiiVerdict(verdict="safe"), _promotion(**overrides)],),patch(f"{_MODULE}.generate_embedding", return_value=_embedding()),patch(f"{_MODULE}.logic.search_knowledge", return_value=[]),@@ -469,7 +483,7 @@ class TestLearningAnalyzer:result = analyze_learning_evidence(input)run.refresh_from_db()- assert result.rejection_code == "unsupported"+ assert result.rejection_code == expected_codeassert run.result == LearningRunResult.NO_KNOWLEDGEpublish.assert_not_called()@@ -521,7 +535,6 @@ class TestLearningAnalyzer:"error_type": "ValidationError","validation_failures": ["canonical_answer:string_too_short",- "rejection_code:literal_error","<other>:extra_forbidden",],},@@ -577,6 +590,34 @@ class TestLearningAnalyzer:client.with_options.assert_called_once_with(timeout=_MODEL_TIMEOUT_SECONDS, max_retries=0)assert entered.messages.create.call_args.kwargs["extra_headers"] == {"X-PostHog-Privacy-Mode": "true"}+ def test_several_extraction_tool_calls_use_the_first_instead_of_failing(self, team: Team) -> None:+ run, input = _setup_sync(team)+ provider = _Provider(EvidenceBundle(replies=("Refunds are available within 30 days.",)))+ client, entered = _gateway_client()+ response = MagicMock()+ response.stop_reason = "tool_use"+ response.content = [+ ToolUseBlock(+ id=f"toolu_{index}",+ name="ExtractedKnowledge",+ input=_extraction(generalizable=generalizable, rejection_code=code).model_dump(),+ type="tool_use",+ )+ for index, (generalizable, code) in enumerate([(False, "case_specific"), (True, "none")])+ ]+ entered.messages.create.return_value = response++ with (+ patch(f"{_MODULE}.get_learning_provider", return_value=provider),+ patch(f"{_MODULE}.build_ai_gateway_anthropic_client", return_value=client),+ ):+ result = analyze_learning_evidence(input)++ run.refresh_from_db()+ assert result.rejection_code == "case_specific"+ assert run.status == "completed"+ assert entered.messages.create.call_count == 1+@pytest.mark.parametrize("content",[