Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix: temporal bk (#124653)

Alex V91231f6f5b6e
diff --git a/products/business_knowledge/backend/temporal/learning/activities/analyze.py b/products/business_knowledge/backend/temporal/learning/activities/analyze.py
index 8bbdc75174f0..7d0f425d84f5 100644
--- a/products/business_knowledge/backend/temporal/learning/activities/analyze.py
+++ b/products/business_knowledge/backend/temporal/learning/activities/analyze.py
@@ -116,6 +116,11 @@ class LearningOutputTruncated(LearningAnalysisError):
super().__init__(f"{stage}_output_truncated")
+class LearningConflictingOutput(LearningAnalysisError):
+ def __init__(self, stage: str) -> None:
+ super().__init__(f"{stage}_conflicting_output")
+
+
class _SupersessionUnavailable(Exception):
"""The conflicting source is not one this job may replace, so the candidate must not be published."""
@@ -202,12 +207,19 @@ def _log_model_failure(team: Team, stage: str, trace_id: str, error: Exception,
)
-def _parse_tool_result(content: object, output_model: type[ModelOutput], stage: str) -> ModelOutput:
+def _parse_tool_result(
+ content: object, output_model: type[ModelOutput], stage: str, *, first_call_wins: bool
+) -> ModelOutput:
tool_name = output_model.__name__
blocks = content if isinstance(content, list) else []
tool_results = [block for block in blocks if isinstance(block, ToolUseBlock) and block.name == tool_name]
- if len(tool_results) != 1:
+ if not tool_results:
raise LearningAnalysisError(f"invalid_{stage}_output")
+ if len(tool_results) > 1:
+ # The request disables parallel tool use. Several calls can still arrive.
+ logger.warning("business_knowledge.learning.multiple_tool_calls", stage=stage, count=len(tool_results))
+ if not first_call_wins and any(block.input != tool_results[0].input for block in tool_results[1:]):
+ raise LearningConflictingOutput(stage)
return output_model.model_validate(tool_results[0].input)
@@ -220,6 +232,7 @@ def _invoke_structured_model(
payload: dict[str, object],
output_model: type[ModelOutput],
max_tokens: int = LEARNING_MAX_TOKENS,
+ first_call_wins: bool = False,
) -> ModelOutput:
try:
client = build_ai_gateway_anthropic_client(
@@ -250,11 +263,13 @@ def _invoke_structured_model(
"input_schema": output_model.model_json_schema(),
}
],
- tool_choice={"type": "tool", "name": tool_name},
+ tool_choice={"type": "tool", "name": tool_name, "disable_parallel_tool_use": True},
extra_headers={"X-PostHog-Privacy-Mode": "true"},
)
try:
- return _parse_tool_result(response.content, output_model, stage)
+ return _parse_tool_result(response.content, output_model, stage, first_call_wins=first_call_wins)
+ except LearningConflictingOutput:
+ raise
except (LearningAnalysisError, ValidationError) as error:
if response.stop_reason == "max_tokens":
logger.warning(
@@ -292,6 +307,8 @@ def _extract_candidate(team: Team, bundle: EvidenceBundle, trace_id: str) -> Ext
system_prompt=_EXTRACTION_SYSTEM_PROMPT,
payload={"public_human_replies": list(bundle.replies)},
output_model=ExtractedKnowledge,
+ # Every later gate checks the candidate this stage returns, so any one of several candidates is safe to keep.
+ first_call_wins=True,
)
@@ -552,6 +569,7 @@ def _log_analysis_failure(run: KnowledgeLearningRun, error: Exception) -> None:
provider=run.provider,
evidence_key=run.evidence_key,
error_type=type(error).__name__,
+ error_code=error.code if isinstance(error, LearningAnalysisError) else None,
)
@@ -736,7 +754,11 @@ def _analyze(run: KnowledgeLearningRun, input: AnalyzeLearningEvidenceInput) ->
return _finish_without_knowledge(run, rejection_code=rejection_code)
generated_text = f"{extracted.canonical_topic}\n{extracted.canonical_answer}"
- if _model_pii_verdict(run.team, extracted, trace_id).verdict != "safe":
+ try:
+ pii_verdict = _model_pii_verdict(run.team, extracted, trace_id)
+ except LearningConflictingOutput:
+ return _finish_without_knowledge(run, rejection_code="pii")
+ if pii_verdict.verdict != "safe":
return _finish_without_knowledge(run, rejection_code="pii")
try:
@@ -744,10 +766,16 @@ def _analyze(run: KnowledgeLearningRun, input: AnalyzeLearningEvidenceInput) ->
except Exception:
raise LearningAnalysisError("knowledge_search_failed") from None
retrieved = _render_search_context(results)
- decision = _promotion_decision(run.team, bundle, extracted, retrieved, trace_id)
+ try:
+ decision = _promotion_decision(run.team, bundle, extracted, retrieved, trace_id)
+ except LearningConflictingOutput:
+ return _finish_without_knowledge(run, rejection_code="low_confidence")
conflicting = _conflicting_search_result(decision, results, len(retrieved))
if conflicting is not None:
- confirmation = _confirm_contradiction(run.team, extracted, conflicting, trace_id)
+ try:
+ confirmation = _confirm_contradiction(run.team, extracted, conflicting, trace_id)
+ except LearningConflictingOutput:
+ return _finish_without_knowledge(run, rejection_code="already_known")
if _is_confirmed_contradiction(confirmation):
if not conflicting.is_generated:
# Learning may retire what it wrote. A source a person wrote stays until a person changes it.
diff --git a/products/business_knowledge/backend/temporal/learning/schemas.py b/products/business_knowledge/backend/temporal/learning/schemas.py
index 8a73ba56efc5..635e5ed27206 100644
--- a/products/business_knowledge/backend/temporal/learning/schemas.py
+++ b/products/business_knowledge/backend/temporal/learning/schemas.py
@@ -1,7 +1,8 @@
from dataclasses import dataclass
-from typing import Literal
+from typing import Literal, get_args
-from pydantic import BaseModel, ConfigDict, Field, StrictBool
+import structlog
+from pydantic import BaseModel, ConfigDict, Field, StrictBool, field_validator
from posthog.dataclasses import frozen
@@ -35,6 +36,17 @@ PromotionRejectionCode = Literal[
"stale_candidate",
]
+logger = structlog.get_logger(__name__)
+
+
+def _known_rejection_code(value: object, allowed: tuple[str, ...], stage: str) -> object:
+ # Tool input does not enforce the schema enum, and temperature 0 returns the same invented code on every retry.
+ # A rejection stops publication, so mapping an unknown code to one fails closed and lets the run complete.
+ if value in allowed:
+ return value
+ logger.warning("business_knowledge.learning.unknown_rejection_code", stage=stage)
+ return "not_useful"
+
@dataclass(frozen=True)
class AnalyzeLearningEvidenceInput:
@@ -87,6 +99,11 @@ class ExtractedKnowledge(BaseModel):
confidence: float = Field(ge=0, le=1)
rejection_code: ExtractionRejectionCode
+ @field_validator("rejection_code", mode="before")
+ @classmethod
+ def _unknown_rejection_code_rejects(cls, value: object) -> object:
+ return _known_rejection_code(value, get_args(ExtractionRejectionCode), "extraction")
+
class PiiVerdict(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
@@ -106,6 +123,11 @@ class PromotionDecision(BaseModel):
confidence: float = Field(ge=0, le=1)
rejection_code: PromotionRejectionCode
+ @field_validator("rejection_code", mode="before")
+ @classmethod
+ def _unknown_rejection_code_rejects(cls, value: object) -> object:
+ return _known_rejection_code(value, get_args(PromotionRejectionCode), "promotion")
+
class ContradictionVerdict(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
diff --git a/products/business_knowledge/backend/tests/test_learning_analyzer.py b/products/business_knowledge/backend/tests/test_learning_analyzer.py
index 281af3fcce71..8aece4038ff3 100644
--- a/products/business_knowledge/backend/tests/test_learning_analyzer.py
+++ b/products/business_knowledge/backend/tests/test_learning_analyzer.py
@@ -180,9 +180,12 @@ def _provider_connection_error() -> Exception:
return error
-def _tool_response(name: str, payload: dict[str, object]) -> MagicMock:
+def _tool_response(name: str, *payloads: dict[str, object]) -> MagicMock:
response = MagicMock()
- response.content = [ToolUseBlock(id="toolu_test", name=name, input=payload, type="tool_use")]
+ response.content = [
+ ToolUseBlock(id=f"toolu_{index}", name=name, input=payload, type="tool_use")
+ for index, payload in enumerate(payloads)
+ ]
return response
@@ -319,6 +322,7 @@ class TestLearningAnalyzer:
"unsupported",
),
({"confidence": 0.4, "rejection_code": "low_confidence"}, "low_confidence"),
+ ({"rejection_code": "contradiction"}, "not_useful"),
],
)
def test_extraction_rejections_never_search_or_publish(
@@ -348,17 +352,21 @@ class TestLearningAnalyzer:
publish.assert_not_called()
@pytest.mark.parametrize(
- "pii_result",
- [PiiVerdict(verdict="unsafe"), PiiVerdict(verdict="uncertain")],
+ "pii_results",
+ [
+ pytest.param([PiiVerdict(verdict="unsafe")], id="unsafe"),
+ pytest.param([PiiVerdict(verdict="uncertain")], id="uncertain"),
+ pytest.param([PiiVerdict(verdict="safe"), PiiVerdict(verdict="unsafe")], id="safe_then_unsafe"),
+ ],
)
- def test_model_pii_rejection_is_fail_closed(self, team: Team, pii_result: PiiVerdict) -> None:
+ def test_model_pii_rejection_is_fail_closed(self, team: Team, pii_results: list[PiiVerdict]) -> None:
run, input = _setup_sync(team)
provider = _Provider(EvidenceBundle(replies=("Use the standard contact process.",)))
client, entered = _gateway_client()
extraction = _extraction(canonical_topic="Contact policy", canonical_answer="Contact Taylor for help.")
entered.messages.create.side_effect = [
_tool_response("ExtractedKnowledge", extraction.model_dump()),
- _tool_response("PiiVerdict", pii_result.model_dump()),
+ _tool_response("PiiVerdict", *(verdict.model_dump() for verdict in pii_results)),
]
with (
@@ -390,6 +398,7 @@ class TestLearningAnalyzer:
for call in entered.messages.create.call_args_list:
assert call.kwargs["extra_headers"] == {"X-PostHog-Privacy-Mode": "true"}
assert call.kwargs["tool_choice"]["type"] == "tool"
+ assert call.kwargs["tool_choice"]["disable_parallel_tool_use"] is True
embed.assert_not_called()
search.assert_not_called()
publish.assert_not_called()
@@ -448,7 +457,16 @@ class TestLearningAnalyzer:
assert search.call_args.kwargs["query_embedding"] == [0.1, 0.2]
publish.assert_not_called()
- def test_promotion_rechecks_evidence_support_before_publication(self, team: Team) -> None:
+ @pytest.mark.parametrize(
+ "overrides,expected_code",
+ [
+ ({"supported_by_public_human_resolution": False, "rejection_code": "unsupported"}, "unsupported"),
+ ({"rejection_code": "contradiction"}, "not_useful"),
+ ],
+ )
+ def test_promotion_rejections_block_publication(
+ self, team: Team, overrides: dict[str, object], expected_code: str
+ ) -> None:
run, input = _setup_sync(team)
provider = _Provider(EvidenceBundle(replies=("Refunds are available within 30 days.",)))
@@ -456,11 +474,7 @@ class TestLearningAnalyzer:
patch(f"{_MODULE}.get_learning_provider", return_value=provider),
patch(
f"{_MODULE}._invoke_structured_model",
- side_effect=[
- _extraction(),
- PiiVerdict(verdict="safe"),
- _promotion(supported_by_public_human_resolution=False, rejection_code="unsupported"),
- ],
+ side_effect=[_extraction(), PiiVerdict(verdict="safe"), _promotion(**overrides)],
),
patch(f"{_MODULE}.generate_embedding", return_value=_embedding()),
patch(f"{_MODULE}.logic.search_knowledge", return_value=[]),
@@ -469,7 +483,7 @@ class TestLearningAnalyzer:
result = analyze_learning_evidence(input)
run.refresh_from_db()
- assert result.rejection_code == "unsupported"
+ assert result.rejection_code == expected_code
assert run.result == LearningRunResult.NO_KNOWLEDGE
publish.assert_not_called()
@@ -521,7 +535,6 @@ class TestLearningAnalyzer:
"error_type": "ValidationError",
"validation_failures": [
"canonical_answer:string_too_short",
- "rejection_code:literal_error",
"<other>:extra_forbidden",
],
},
@@ -577,6 +590,34 @@ class TestLearningAnalyzer:
client.with_options.assert_called_once_with(timeout=_MODEL_TIMEOUT_SECONDS, max_retries=0)
assert entered.messages.create.call_args.kwargs["extra_headers"] == {"X-PostHog-Privacy-Mode": "true"}
+ def test_several_extraction_tool_calls_use_the_first_instead_of_failing(self, team: Team) -> None:
+ run, input = _setup_sync(team)
+ provider = _Provider(EvidenceBundle(replies=("Refunds are available within 30 days.",)))
+ client, entered = _gateway_client()
+ response = MagicMock()
+ response.stop_reason = "tool_use"
+ response.content = [
+ ToolUseBlock(
+ id=f"toolu_{index}",
+ name="ExtractedKnowledge",
+ input=_extraction(generalizable=generalizable, rejection_code=code).model_dump(),
+ type="tool_use",
+ )
+ for index, (generalizable, code) in enumerate([(False, "case_specific"), (True, "none")])
+ ]
+ entered.messages.create.return_value = response
+
+ with (
+ patch(f"{_MODULE}.get_learning_provider", return_value=provider),
+ patch(f"{_MODULE}.build_ai_gateway_anthropic_client", return_value=client),
+ ):
+ result = analyze_learning_evidence(input)
+
+ run.refresh_from_db()
+ assert result.rejection_code == "case_specific"
+ assert run.status == "completed"
+ assert entered.messages.create.call_count == 1
+
@pytest.mark.parametrize(
"content",
[